/**
* @return Scalr_Net_Ssh2_Client
* Enter description here ...
*/
public function GetSsh2Client()
{
$ssh2Client = new Scalr_Net_Ssh2_Client();
switch ($this->platform) {
case SERVER_PLATFORMS::RACKSPACE:
$ssh2Client->addPassword('root', $this->GetProperty(RACKSPACE_SERVER_PROPERTIES::ADMIN_PASS));
break;
case SERVER_PLATFORMS::EC2:
$userName = '******';
// Temporary server for role builder
if ($this->status == SERVER_STATUS::TEMPORARY) {
$keyName = 'SCALR-ROLESBUILDER';
} else {
$keyName = "FARM-{$this->farmId}";
}
try {
$key = Scalr_Model::init(Scalr_Model::SSH_KEY)->loadGlobalByName($keyName, $this->GetProperty(EC2_SERVER_PROPERTIES::REGION), $this->envId);
if (!$key) {
throw new Exception(_("There is no SSH key for server: {$this->serverId}"));
}
} catch (Exception $e) {
throw new Exception("Cannot init SshKey object: {$e->getMessage()}");
}
$priv_key_file = tempnam("/tmp", "AWSK");
@file_put_contents($priv_key_file, $key->getPrivate());
$pub_key_file = tempnam("/tmp", "AWSK");
@file_put_contents($pub_key_file, $key->getPublic());
$ssh2Client->addPubkey($userName, $pub_key_file, $priv_key_file);
break;
}
return $ssh2Client;
}
public function rndcStatus()
{
$retval = $this->ssh2Client->exec("{$this->rndcPath} status");
$this->logger->info("Execute rndc status: {$retval}");
preg_match_all("/number of zones:[^0-9]([0-9]+)/", $retval, $matches);
if ($matches[1][0] > 0) {
return $matches[1][0];
} else {
return false;
}
}
/**
* Upload S3cmd config file, AWS private key and certificate to instance aftre instance boot.
* Also execute hostInit hooks from hooks/hostInit folder
*
* @param array $instanceinfo
* @param string $local_ip
* @param string $remote_ip
* @param string $public_key
*/
public function OnHostInit(HostInitEvent $event)
{
if ($event->DBServer->IsSupported("0.5")) {
$this->Logger->info("Scalarizr instance. Skipping SSH observer...");
return true;
}
if ($event->DBServer->platform != SERVER_PLATFORMS::EC2) {
return true;
}
// Get farm info and client info from database;
$DBFarm = DBFarm::LoadByID($this->FarmID);
$DBRole = DBRole::loadById($event->DBServer->roleId);
// Get Role info
$ssh_port = $DBRole->getProperty(DBRole::PROPERTY_SSH_PORT) ? $DBRole->getProperty(DBRole::PROPERTY_SSH_PORT) : 22;
// Generate s3cmd config file
$s3cfg = CONFIG::$S3CFG_TEMPLATE;
$s3cfg = str_replace("[access_key]", $DBFarm->GetEnvironmentObject()->getPlatformConfigValue(Modules_Platforms_Ec2::ACCESS_KEY), $s3cfg);
$s3cfg = str_replace("[secret_key]", $DBFarm->GetEnvironmentObject()->getPlatformConfigValue(Modules_Platforms_Ec2::SECRET_KEY), $s3cfg);
$s3cfg = str_replace("\r\n", "\n", $s3cfg);
// Prepare public key for SSH connection
$pub_key_file = tempnam("/tmp", "AWSK");
$res = file_put_contents($pub_key_file, $event->PublicKey);
$this->Logger->debug("Creating temporary file for public key: {$res}");
try {
$key = Scalr_Model::init(Scalr_Model::SSH_KEY)->loadGlobalByFarmId($event->DBServer->farmId, $event->DBServer->GetFarmRoleObject()->CloudLocation);
if (!$key) {
throw new Exception(_("There is no SSH key for server: {$event->DBServer->serverId}"));
}
} catch (Exception $e) {
throw new Exception("Cannot init SshKey object: {$e->getMessage()}");
}
// Prepare private key for SSH connection
$priv_key_file = tempnam("/tmp", "AWSK");
$res = file_put_contents($priv_key_file, $key->getPrivate());
$this->Logger->debug("Creating temporary file for private key: {$res}");
// Connect to SSH
$SSH2 = new Scalr_Net_Ssh2_Client();
$SSH2->addPubkey("root", $pub_key_file, $priv_key_file);
if ($SSH2->connect($event->ExternalIP, $ssh_port)) {
// Upload keys and s3 config to instance
$res = $SSH2->sendFile("/etc/aws/keys/pk.pem", $DBFarm->GetEnvironmentObject()->getPlatformConfigValue(Modules_Platforms_Ec2::PRIVATE_KEY), "w+", false);
$res2 = $SSH2->sendFile("/etc/aws/keys/cert.pem", $DBFarm->GetEnvironmentObject()->getPlatformConfigValue(Modules_Platforms_Ec2::CERTIFICATE), "w+", false);
$res3 = $SSH2->sendFile("/etc/aws/keys/s3cmd.cfg", $s3cfg, "w+", false);
// remove temporary files
@unlink($pub_key_file);
@unlink($priv_key_file);
} else {
// remove temporary files
@unlink($pub_key_file);
@unlink($priv_key_file);
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($this->FarmID, "Cannot upload ec2 keys to '{$event->DBServer->serverId}' instance. Failed to connect to SSH '{$event->ExternalIP}:{$ssh_port}'"));
throw new Exception("Cannot upload keys on '{$event->DBServer->serverId}'. Failed to connect to '{$event->ExternalIP}:{$ssh_port}'.");
}
}
/**
* @return Scalr_Net_Ssh2_Client
* Enter description here ...
*/
public function GetSsh2Client()
{
$ssh2Client = new Scalr_Net_Ssh2_Client();
switch ($this->platform) {
case SERVER_PLATFORMS::RACKSPACENG_UK:
case SERVER_PLATFORMS::RACKSPACENG_US:
$ssh2Client->addPassword('root', $this->GetProperty(OPENSTACK_SERVER_PROPERTIES::ADMIN_PASS));
break;
case SERVER_PLATFORMS::RACKSPACE:
$ssh2Client->addPassword('root', $this->GetProperty(RACKSPACE_SERVER_PROPERTIES::ADMIN_PASS));
break;
case SERVER_PLATFORMS::GCE:
$userName = '******';
if ($this->status == SERVER_STATUS::TEMPORARY) {
$keyName = 'SCALR-ROLESBUILDER-' . SCALR_ID;
} else {
$keyName = "FARM-{$this->farmId}-" . SCALR_ID;
}
try {
$key = (new SshKey())->loadGlobalByName($this->envId, SERVER_PLATFORMS::GCE, "", $keyName);
if (!$key) {
throw new Exception(_("There is no SSH key for server: {$this->serverId}"));
}
} catch (Exception $e) {
throw new Exception("Cannot init SshKey object: {$e->getMessage()}");
}
$priv_key_file = tempnam("/tmp", "GCEPK");
@file_put_contents($priv_key_file, $key->privateKey);
$this->tmpFiles[] = $priv_key_file;
$pub_key_file = tempnam("/tmp", "GCEK");
@file_put_contents($pub_key_file, $key->publicKey);
$this->tmpFiles[] = $pub_key_file;
$ssh2Client->addPubkey($userName, $pub_key_file, $priv_key_file);
break;
case SERVER_PLATFORMS::IDCF:
case SERVER_PLATFORMS::EC2:
$userName = '******';
$skipKeyValidation = false;
// Temporary server for role builder
$sshKey = new SshKey();
if ($this->status == SERVER_STATUS::TEMPORARY) {
$keyName = "SCALR-ROLESBUILDER-" . SCALR_ID . "-{$this->envId}";
if (!$sshKey->loadGlobalByName($this->envId, $this->platform, $this->GetCloudLocation(), $keyName)) {
$keyName = "SCALR-ROLESBUILDER-" . SCALR_ID;
}
try {
$bundleTaskId = $this->GetProperty(\SERVER_PROPERTIES::SZR_IMPORTING_BUNDLE_TASK_ID);
$bundleTask = BundleTask::LoadById($bundleTaskId);
if ($bundleTask->osFamily == 'amazon') {
$userName = '******';
}
} catch (Exception $e) {
}
} else {
$keyName = "FARM-{$this->farmId}-" . SCALR_ID;
$oldKeyName = "FARM-{$this->farmId}";
$key = $sshKey->loadGlobalByName($this->envId, $this->platform, $this->GetCloudLocation(), $oldKeyName);
if ($key) {
$keyName = $oldKeyName;
$skipKeyValidation = true;
}
}
if (!$skipKeyValidation) {
try {
$key = $sshKey->loadGlobalByName($this->envId, $this->platform, $this->GetCloudLocation(), $keyName);
if (!$key) {
throw new Exception(sprintf('Could not find SSH Key for server "%s" with name:"%s", cloud-location:"%s", platform:"%s", environment:"%d".', $this->serverId, $keyName, $this->GetCloudLocation(), $this->platform, $this->envId));
}
} catch (Exception $e) {
throw new Exception("Cannot init SshKey object: {$e->getMessage()}");
}
}
$priv_key_file = tempnam("/tmp", "AWSK");
@file_put_contents($priv_key_file, $key->privateKey);
$this->tmpFiles[] = $priv_key_file;
$pub_key_file = tempnam("/tmp", "AWSK");
$this->tmpFiles[] = $pub_key_file;
$pubKey = $key->publicKey;
if (!stristr($pubKey, $keyName)) {
$pubKey .= " {$keyName}";
}
@file_put_contents($pub_key_file, $pubKey);
$ssh2Client->addPubkey($userName, $pub_key_file, $priv_key_file);
break;
}
return $ssh2Client;
}
function handleWork($farmId)
{
$this->cleanup();
$DBFarm = DBFarm::LoadByID($farmId);
$GLOBALS["SUB_TRANSACTIONID"] = abs(crc32(posix_getpid() . $farmId));
$GLOBALS["LOGGER_FARMID"] = $farmId;
$this->logger->info("[" . $GLOBALS["SUB_TRANSACTIONID"] . "] Begin polling farm (ID: {$DBFarm->ID}, Name: {$DBFarm->Name}, Status: {$DBFarm->Status})");
//
// Collect information from database
//
$servers_count = $this->db->GetOne("SELECT COUNT(*) FROM servers WHERE farm_id = ? AND status != ?", array($DBFarm->ID, SERVER_STATUS::TERMINATED));
$this->logger->info("[FarmID: {$DBFarm->ID}] Found {$servers_count} farm instances in database");
if ($DBFarm->Status == FARM_STATUS::TERMINATED && $servers_count == 0) {
return;
}
/*
O:8:"stdClass":1:{s:7:"servers";a:3:{i:0;O:8:"stdClass":9:{s:8:"progress";i:100;s:2:"id";i:20842485;s:7:"imageId";i:77;s:8:"flavorId";i:3;s:6:"status";s:6:"ACTIVE";s:4:"name";s:36:"4d63c9c4-6f2d-4c16-9c1e-623ef2b70da8";s:6:"hostId";s:32:"1de22da589d35371f10ef232b77a6ede";s:9:"addresses";O:8:"stdClass":2:{s:6:"public";a:1:{i:0;s:13:"50.57.122.122";}s:7:"private";a:1:{i:0;s:13:"10.182.160.86";}}s:8:"metadata";O:8:"stdClass":0:{}}i:1;O:8:"stdClass":9:{s:8:"progress";i:100;s:2:"id";i:20842486;s:7:"imageId";i:77;s:8:"flavorId";i:3;s:6:"status";s:6:"ACTIVE";s:4:"name";s:36:"df6f3f67-0cfb-482d-9307-3d394dd009a1";s:6:"hostId";s:32:"00a70eb9f5b7e667d4f1ff3d8c5ef496";s:9:"addresses";O:8:"stdClass":2:{s:6:"public";a:1:{i:0;s:13:"50.57.122.131";}s:7:"private";a:1:{i:0;s:13:"10.182.160.89";}}s:8:"metadata";O:8:"stdClass":0:{}}i:2;O:8:"stdClass":9:{s:8:"progress";i:100;s:2:"id";i:20842516;s:7:"imageId";i:77;s:8:"flavorId";i:3;s:6:"status";s:6:"ACTIVE";s:4:"name";s:36:"27128bc2-1da8-4cef-b527-0b916309ae7e";s:6:"hostId";s:32:"254b44d9672258f1d7d189ab85627842";s:9:"addresses";O:8:"stdClass":2:{s:6:"public";a:1:{i:0;s:13:"50.57.122.140";}s:7:"private";a:1:{i:0;s:14:"10.182.160.112";}}s:8:"metadata";O:8:"stdClass":0:{}}}}
O:8:"stdClass":1:{s:7:"servers";a:3:{i:0;O:8:"stdClass":9:{s:8:"progress";i:100;s:2:"id";i:20842485;s:7:"imageId";i:77;s:8:"flavorId";i:3;s:6:"status";s:6:"ACTIVE";s:4:"name";s:36:"4d63c9c4-6f2d-4c16-9c1e-623ef2b70da8";s:6:"hostId";s:32:"1de22da589d35371f10ef232b77a6ede";s:9:"addresses";O:8:"stdClass":2:{s:6:"public";a:1:{i:0;s:13:"50.57.122.122";}s:7:"private";a:1:{i:0;s:13:"10.182.160.86";}}s:8:"metadata";O:8:"stdClass":0:{}}i:1;O:8:"stdClass":9:{s:8:"progress";i:100;s:2:"id";i:20842486;s:7:"imageId";i:77;s:8:"flavorId";i:3;s:6:"status";s:6:"ACTIVE";s:4:"name";s:36:"df6f3f67-0cfb-482d-9307-3d394dd009a1";s:6:"hostId";s:32:"00a70eb9f5b7e667d4f1ff3d8c5ef496";s:9:"addresses";O:8:"stdClass":2:{s:6:"public";a:1:{i:0;s:13:"50.57.122.131";}s:7:"private";a:1:{i:0;s:13:"10.182.160.89";}}s:8:"metadata";O:8:"stdClass":0:{}}i:2;O:8:"stdClass":9:{s:8:"progress";i:100;s:2:"id";i:20842516;s:7:"imageId";i:77;s:8:"flavorId";i:3;s:6:"status";s:6:"ACTIVE";s:4:"name";s:36:"27128bc2-1da8-4cef-b527-0b916309ae7e";s:6:"hostId";s:32:"254b44d9672258f1d7d189ab85627842";s:9:"addresses";O:8:"stdClass":2:{s:6:"public";a:1:{i:0;s:13:"50.57.122.140";}s:7:"private";a:1:{i:0;s:14:"10.182.160.112";}}s:8:"metadata";O:8:"stdClass":0:{}}}}
*/
foreach ($DBFarm->GetServersByFilter(array(), array('status' => SERVER_STATUS::PENDING_LAUNCH)) as $DBServer) {
try {
if ($DBServer->status != SERVER_STATUS::PENDING) {
$p = PlatformFactory::NewPlatform($DBServer->platform);
if (!$p->IsServerExists($DBServer)) {
if ($DBServer->platform == SERVER_PLATFORMS::RACKSPACE) {
if ($DBServer->status != SERVER_STATUS::PENDING_TERMINATE && $DBServer->status != SERVER_STATUS::TERMINATED) {
//TODO:
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' found in database but not found on {$DBServer->platform}. Crashed. RACKSPACE, doing nothing.", $DBServer->serverId)));
mail("*****@*****.**", "RACKSPACE - {$DBServer->serverId} ({$DBServer->remoteIp}) ({$DBServer->GetCloudServerID()}) ({$DBServer->GetProperty(RACKSPACE_SERVER_PROPERTIES::HOST_ID)})", serialize($p->_tmpVar));
$DBServer->SetProperty("rackspace.crashed", "1");
continue;
}
} else {
if ($DBServer->status != SERVER_STATUS::TERMINATED && $DBServer->status != SERVER_STATUS::PENDING_TERMINATE) {
$DBServer->SetProperty(SERVER_PROPERTIES::REBOOTING, 0);
// Add entry to farm log
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' found in database but not found on {$DBServer->platform}. Crashed.", $DBServer->serverId)));
Scalr::FireEvent($DBFarm->ID, new HostCrashEvent($DBServer));
continue;
}
}
}
}
} catch (Exception $e) {
if (stristr($e->getMessage(), "AWS was not able to validate the provided access credentials")) {
continue;
}
if (stristr($e->getMessage(), "Could not connect to host")) {
continue;
}
print "[Farm: {$farmId}] {$e->getMessage()} at {$e->getFile()}:{$e->getLine()}\n\n";
continue;
}
try {
if ($DBServer->status != SERVER_STATUS::TERMINATED && $DBServer->GetRealStatus()->isTerminated()) {
if ($DBServer->status != SERVER_STATUS::PENDING_TERMINATE) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) not running (Real state: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName())));
}
$DBServer->SetProperty(SERVER_PROPERTIES::REBOOTING, 0);
Scalr::FireEvent($DBFarm->ID, new HostDownEvent($DBServer));
continue;
} elseif ($DBServer->GetRealStatus()->IsRunning() && $DBServer->status != SERVER_STATUS::RUNNING) {
if ($DBServer->status != SERVER_STATUS::TERMINATED) {
if ($DBServer->platform == SERVER_PLATFORMS::RDS) {
//TODO: timeouts
if ($DBServer->status == SERVER_STATUS::PENDING) {
$info = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
$event = new HostInitEvent($DBServer, $info['localIp'], $info['remoteIp'], '');
} elseif ($DBServer->status == SERVER_STATUS::INIT) {
$event = new HostUpEvent($DBServer, "");
// TODO: add mysql replication password
}
if ($event) {
Scalr::FireEvent($DBServer->farmId, $event);
} else {
//TODO: Log
}
} else {
if ($DBServer->platform == SERVER_PLATFORMS::NIMBULA) {
if (!$DBServer->GetProperty(NIMBULA_SERVER_PROPERTIES::USER_DATA_INJECTED)) {
$dbRole = $DBServer->GetFarmRoleObject()->GetRoleObject();
$ssh2Client = new Scalr_Net_Ssh2_Client();
$ssh2Client->addPassword($dbRole->getProperty(DBRole::PROPERTY_NIMBULA_INIT_ROOT_USER), $dbRole->getProperty(DBRole::PROPERTY_NIMBULA_INIT_ROOT_PASS));
$info = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
$port = $dbRole->getProperty(DBRole::PROPERTY_SSH_PORT);
if (!$port) {
$port = 22;
}
try {
$ssh2Client->connect($info['remoteIp'], $port);
foreach ($DBServer->GetCloudUserData() as $k => $v) {
$u_data .= "{$k}={$v};";
}
$u_data = trim($u_data, ";");
$ssh2Client->sendFile('/etc/scalr/private.d/.user-data', $u_data, "w+", false);
$DBServer->SetProperty(NIMBULA_SERVER_PROPERTIES::USER_DATA_INJECTED, 1);
} catch (Exception $e) {
Logger::getLogger(LOG_CATEGORY::FARM)->error(new FarmLogMessage($DBFarm->ID, $e->getMessage()));
}
}
}
try {
$dtadded = strtotime($DBServer->dateAdded);
$DBFarmRole = $DBServer->GetFarmRoleObject();
$launch_timeout = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) > 0 ? $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) : 300;
} catch (Exception $e) {
if (stristr($e->getMessage(), "not found")) {
PlatformFactory::NewPlatform($DBServer->platform)->TerminateServer($DBServer);
$DBServer->status = SERVER_STATUS::TERMINATED;
$DBServer->Save();
}
}
$scripting_event = false;
if ($DBServer->status == SERVER_STATUS::PENDING) {
$event = "hostInit";
$scripting_event = EVENT_TYPE::HOST_INIT;
} elseif ($DBServer->status == SERVER_STATUS::INIT) {
$event = "hostUp";
$scripting_event = EVENT_TYPE::HOST_UP;
}
if ($scripting_event) {
$scripting_timeout = (int) $this->db->GetOne("SELECT sum(timeout) FROM farm_role_scripts \n\t\t\t\t\t\t\t\t\t\tWHERE event_name=? AND \n\t\t\t\t\t\t\t\t\t\tfarm_roleid=? AND issync='1'", array($scripting_event, $DBServer->farmRoleId));
if ($scripting_timeout) {
$launch_timeout = $launch_timeout + $scripting_timeout;
}
if ($dtadded + $launch_timeout < time()) {
//Add entry to farm log
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, "Server '{$DBServer->serverId}' did not send '{$event}' event in {$launch_timeout} seconds after launch (Try increasing timeouts in role settings). Considering it broken. Terminating instance."));
try {
Scalr::FireEvent($DBFarm->ID, new BeforeHostTerminateEvent($DBServer, false));
} catch (Exception $err) {
$this->logger->fatal($err->getMessage());
}
}
}
// Is IP address changed?
if (!$DBServer->IsRebooting()) {
$ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
if ($ipaddresses['remoteIp'] && $DBServer->remoteIp != $ipaddresses['remoteIp']) {
Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp']));
}
//TODO: Check health:
}
}
}
} elseif ($DBServer->GetRealStatus()->isRunning() && $DBServer->status == SERVER_STATUS::RUNNING) {
// Is IP address changed?
if (!$DBServer->IsRebooting()) {
$ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
if ($ipaddresses['remoteIp'] && $DBServer->remoteIp != $ipaddresses['remoteIp']) {
Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp']));
}
//TODO: Check health:
} else {
//TODO: Check reboot timeout
}
}
if ($DBServer->status == SERVER_STATUS::PENDING_TERMINATE || $DBServer->status == SERVER_STATUS::TERMINATED) {
if ($DBServer->status == SERVER_STATUS::TERMINATED || !$DBServer->dateShutdownScheduled || $DBServer->dateShutdownScheduled && strtotime($DBServer->dateShutdownScheduled) + 60 * 3 < time()) {
try {
if (!$DBServer->GetRealStatus()->isTerminated()) {
try {
if ($DBServer->GetFarmRoleObject()->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ)) {
$serversCount = count($DBServer->GetFarmRoleObject()->GetServersByFilter(array(), array('status' => SERVER_STATUS::TERMINATED)));
if ($DBServer->index == 1 && $serversCount > 1) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("RabbitMQ role. Main DISK node should be terminated after all other nodes. Waiting... (Platform: %s) (Poller).", $DBServer->serverId, $DBServer->platform)));
continue;
}
}
} catch (Exception $e) {
}
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Terminating server '%s' (Platform: %s) (Poller).", $DBServer->serverId, $DBServer->platform)));
PlatformFactory::NewPlatform($DBServer->platform)->TerminateServer($DBServer);
$this->db->Execute("UPDATE servers_history SET\n\t\t\t\t\t\t\t\t\t\tdtterminated\t= NOW(),\n\t\t\t\t\t\t\t\t\t\tterminate_reason\t= ?\n\t\t\t\t\t\t\t\t\t\tWHERE server_id = ?\n\t\t\t\t\t\t\t\t\t", array(sprintf("Server is running on Amazon but has terminated status on Scalr"), $DBServer->serverId));
}
} catch (Exception $e) {
if (stristr($e->getMessage(), "not found")) {
$this->db->Execute("UPDATE servers_history SET\n\t\t\t\t\t\t\t\t\t\tdtterminated\t= NOW(),\n\t\t\t\t\t\t\t\t\t\tterminate_reason\t= ?\n\t\t\t\t\t\t\t\t\t\tWHERE server_id = ?\n\t\t\t\t\t\t\t\t\t", array(sprintf("Role was removed from farm"), $DBServer->serverId));
$DBServer->Remove();
} elseif (stristr($e->getMessage(), "disableApiTermination")) {
continue;
} else {
throw $e;
}
}
}
}
} catch (Exception $e) {
if (stristr($e->getMessage(), "not found")) {
var_dump($e);
} else {
print "[Farm: {$farmId}] {$e->getMessage()} at {$e->getFile()}:{$e->getLine()}\n\n";
}
}
}
}
