public function ServerGetExtendedInformation($ServerID)
{
$DBServer = DBServer::LoadByID($ServerID);
if ($DBServer->envId != $this->Environment->id) {
throw new Exception(sprintf("Server ID #%s not found", $ServerID));
}
$response = $this->CreateInitialResponse();
$info = PlatformFactory::NewPlatform($DBServer->platform)->GetServerExtendedInformation($DBServer);
$response->ServerInfo = new stdClass();
$scalrProps = array('ServerID' => $DBServer->serverId, 'Platform' => $DBServer->platform, 'RemoteIP' => $DBServer->remoteIp ? $DBServer->remoteIp : '', 'LocalIP' => $DBServer->localIp ? $DBServer->localIp : '', 'Status' => $DBServer->status, 'Index' => $DBServer->index, 'AddedAt' => $DBServer->dateAdded);
foreach ($scalrProps as $k => $v) {
$response->ServerInfo->{$k} = $v;
}
$response->PlatformProperties = new stdClass();
if (is_array($info) && count($info)) {
foreach ($info as $name => $value) {
$name = str_replace(".", "_", $name);
$name = preg_replace("/[^A-Za-z0-9_-]+/", "", $name);
if ($name == 'MonitoringCloudWatch') {
continue;
}
$response->PlatformProperties->{$name} = $value;
}
}
$response->ScalrProperties = new stdClass();
if (count($DBServer->GetAllProperties())) {
$it = array();
foreach ($DBServer->GetAllProperties() as $name => $value) {
$name = preg_replace("/[^A-Za-z0-9-]+/", "", $name);
$response->ScalrProperties->{$name} = $value;
}
}
return $response;
}
public function xListVolumesAction()
{
$this->request->defineParams(array('sort' => array('type' => 'json', 'default' => array('property' => 'volumeId', 'direction' => 'ASC')), 'volumeId'));
$platformName = $this->getParam('platform');
if (!$platformName) {
throw new Exception("Cloud should be specified");
}
$platform = PlatformFactory::NewPlatform($platformName);
$cs = Scalr_Service_Cloud_Cloudstack::newCloudstack($platform->getConfigVariable(Modules_Platforms_Cloudstack::API_URL, $this->environment), $platform->getConfigVariable(Modules_Platforms_Cloudstack::API_KEY, $this->environment), $platform->getConfigVariable(Modules_Platforms_Cloudstack::SECRET_KEY, $this->environment), $platformName);
$volumes = $cs->listVolumes($this->getParam('cloudLocation'), 1);
$vols = array();
foreach ($volumes as $pk => $pv) {
if ($this->getParam('volumeId') && $this->getParam('volumeId') != $pv->id) {
continue;
}
$item = array('volumeId' => $pv->id, 'size' => round($pv->size / 1024 / 1024 / 1024, 2), 'status' => $pv->state, 'attachmentStatus' => $pv->virtualmachineid ? 'attached' : 'available', 'device' => $pv->deviceid, 'instanceId' => $pv->virtualmachineid, 'type' => $pv->type . " ({$pv->storagetype})", 'storage' => $pv->storage);
$item['autoSnaps'] = $this->db->GetOne("SELECT id FROM autosnap_settings WHERE objectid=? AND object_type=? LIMIT 1", array($pv->id, AUTOSNAPSHOT_TYPE::CSVOL)) ? true : false;
if ($item['instanceId']) {
try {
$dbServer = DBServer::LoadByPropertyValue(CLOUDSTACK_SERVER_PROPERTIES::SERVER_ID, $item['instanceId']);
$item['farmId'] = $dbServer->farmId;
$item['farmRoleId'] = $dbServer->farmRoleId;
$item['serverIndex'] = $dbServer->index;
$item['serverId'] = $dbServer->serverId;
$item['farmName'] = $dbServer->GetFarmObject()->Name;
$item['mountStatus'] = false;
$item['roleName'] = $dbServer->GetFarmRoleObject()->GetRoleObject()->name;
} catch (Exception $e) {
}
}
$vols[] = $item;
}
$response = $this->buildResponseFromData($vols, array('serverId', 'volumeId', 'farmId', 'farmRoleId', 'storage'));
$this->response->data($response);
}
public function deleteAction()
{
$this->request->defineParams(array('sshKeyId' => array('type' => 'int')));
$sshKey = Scalr_SshKey::init()->loadById($this->getParam('sshKeyId'));
$this->user->getPermissions()->validate($sshKey);
if ($sshKey->type == Scalr_SshKey::TYPE_GLOBAL) {
if ($sshKey->platform == SERVER_PLATFORMS::EC2) {
$aws = $this->getEnvironment()->aws($sshKey->cloudLocation);
$aws->ec2->keyPair->delete($sshKey->cloudKeyName);
$sshKey->delete();
$this->response->success();
} elseif (PlatformFactory::isOpenstack($sshKey->platform)) {
$os = $this->getEnvironment()->openstack($sshKey->platform, $sshKey->cloudLocation);
try {
$os->servers->keypairs->delete($sshKey->cloudKeyName);
} catch (Exception $e) {
}
$sshKey->delete();
$this->response->success();
} else {
$sshKey->delete();
}
} else {
//TODO:
}
$this->response->success("SSH key successfully removed");
}
public function xGrowStorageAction()
{
$dbFarmRole = DBFarmRole::LoadByID($this->getParam('farmRoleId'));
$this->user->getPermissions()->validate($dbFarmRole->GetFarmObject());
$behavior = Scalr_Role_Behavior::loadByName($dbFarmRole->GetRoleObject()->getDbMsrBehavior());
$master = $behavior->getMasterServer($dbFarmRole);
if ($master) {
$port = $master->GetProperty(SERVER_PROPERTIES::SZR_API_PORT);
if (!$port) {
$port = 8010;
}
try {
$client = Scalr_Net_Scalarizr_Client::getClient($master, Scalr_Net_Scalarizr_Client::NAMESPACE_MYSQL, $port);
$volume = Scalr_Storage_Volume::init()->loadById($dbFarmRole->GetSetting(Scalr_Db_Msr::VOLUME_ID));
if ($volume->type != MYSQL_STORAGE_ENGINE::EBS && $volume->type != MYSQL_STORAGE_ENGINE::RAID_EBS && $volume->type != 'raid') {
throw new Exception("Grow feature available only for EBS and RAID storage types");
}
if ($volume->size >= (int) $this->getParam('newSize')) {
throw new Exception("New size should be greather than current one ({$volume->size} GB)");
}
$volumeConfig = $volume->getConfig();
$platformAccessData = PlatformFactory::NewPlatform($dbFarmRole->Platform)->GetPlatformAccessData($this->environment, $master);
$result = $client->growStorage($volumeConfig, $this->getParam('newSize'), $platformAccessData);
// Do not remove. We need to wait a bit before operation will be registered in scalr.
sleep(2);
$this->response->data(array('operationId' => $result));
} catch (Exception $e) {
throw new Exception("Cannot grow storage: {$e->getMessage()}");
}
} else {
throw new Exception("Impossible to increase storage size. No running master server.");
}
}
public static function getAwsLocations()
{
$locations = array();
foreach (PlatformFactory::NewPlatform(SERVER_PLATFORMS::EC2)->getLocations() as $key => $loc) {
$locations[] = array($key, $loc);
}
return $locations;
}
public function viewAction()
{
if ($this->getParam('platform')) {
$locations = self::loadController('Platforms')->getCloudLocations(array($this->getParam('platform')), false);
} else {
$locations = self::loadController('Platforms')->getCloudLocations(PlatformFactory::getOpenstackBasedPlatforms(), false);
}
$this->response->page('ui/tools/openstack/volumes/view.js', array('locations' => $locations));
}
public function getEnabledPlatforms($addLocations = false, $includeGCELocations = true)
{
$ePlatforms = $this->user->isScalrAdmin() ? array_keys(SERVER_PLATFORMS::GetList()) : $this->getEnvironment()->getEnabledPlatforms();
$lPlatforms = SERVER_PLATFORMS::GetList();
$platforms = array();
foreach ($ePlatforms as $platform) {
$platforms[$platform] = $addLocations ? array('id' => $platform, 'name' => $lPlatforms[$platform], 'locations' => $platform !== SERVER_PLATFORMS::GCE || $includeGCELocations ? PlatformFactory::NewPlatform($platform)->getLocations() : array()) : $lPlatforms[$platform];
}
return $platforms;
}
public function getEnabledPlatforms($addLocations = false)
{
$ePlatforms = $this->getEnvironment()->getEnabledPlatforms();
$lPlatforms = SERVER_PLATFORMS::GetList();
$platforms = array();
foreach ($ePlatforms as $platform) {
$platforms[$platform] = $addLocations ? array('id' => $platform, 'name' => $lPlatforms[$platform], 'locations' => PlatformFactory::NewPlatform($platform)->getLocations()) : $lPlatforms[$platform];
}
return $platforms;
}
public function xGetMigrateDetailsAction()
{
if (!$this->request->getEnvironment()->isPlatformEnabled(SERVER_PLATFORMS::EC2)) {
throw new Exception('You can migrate image between regions only on EC2 cloud');
}
$platform = PlatformFactory::NewPlatform(SERVER_PLATFORMS::EC2);
$locationsList = $platform->getLocations();
foreach ($locationsList as $location => $name) {
if ($location != $this->getParam('cloudLocation')) {
$availableDestinations[] = array('cloudLocation' => $location, 'name' => $name);
}
}
$this->response->data(array('sourceRegion' => $this->getParam('cloudLocation'), 'availableDestinations' => $availableDestinations, 'snapshotId' => $this->getParam('snapshotId')));
}
public function xListIpsAction()
{
$this->request->defineParams(array('sort' => array('type' => 'json', 'default' => array('property' => 'ipId', 'direction' => 'ASC')), 'ipId'));
$platformName = $this->getParam('platform');
if (!$platformName) {
throw new Exception("Cloud should be specified");
}
$platform = PlatformFactory::NewPlatform($platformName);
$cs = Scalr_Service_Cloud_Cloudstack::newCloudstack($platform->getConfigVariable(Modules_Platforms_Cloudstack::API_URL, $this->environment), $platform->getConfigVariable(Modules_Platforms_Cloudstack::API_KEY, $this->environment), $platform->getConfigVariable(Modules_Platforms_Cloudstack::SECRET_KEY, $this->environment), $platformName);
$accountName = $platform->getConfigVariable(Modules_Platforms_Cloudstack::ACCOUNT_NAME, $this->getEnvironment(), false);
$domainId = $platform->getConfigVariable(Modules_Platforms_Cloudstack::DOMAIN_ID, $this->getEnvironment(), false);
$ipAddresses = $cs->listPublicIpAddresses(null, $accountName, null, $domainId, null, null, null, null, null, null, $this->getParam('cloudLocation'));
$systemIp = $platform->getConfigVariable(Modules_Platforms_Cloudstack::SHARED_IP . "." . $this->getParam('cloudLocation'), $this->environment);
$ips = array();
foreach ($ipAddresses->publicipaddress as $pk => $pv) {
if ($this->getParam('ipId') && $this->getParam('ipId') != $pv->id) {
continue;
}
if ($pv->ipaddress == $systemIp) {
$pv->purpose = 'ScalrShared';
}
if ($pv->isstaticnat && !$pv->issystem) {
$pv->purpose = 'ElasticIP';
}
if ($pv->isstaticnat && $pv->issystem) {
$pv->purpose = 'PublicIP';
}
$item = array('ipId' => $pv->id, 'dtAllocated' => $pv->allocated, 'networkName' => $pv->associatednetworkname, 'purpose' => $pv->purpose ? $pv->purpose : "Not used", 'ip' => $pv->ipaddress, 'state' => $pv->state, 'instanceId' => $pv->virtualmachineid, 'fullinfo' => $pv, 'farmId' => false);
if ($item['instanceId']) {
try {
$dbServer = DBServer::LoadByPropertyValue(CLOUDSTACK_SERVER_PROPERTIES::SERVER_ID, $item['instanceId']);
$item['farmId'] = $dbServer->farmId;
$item['farmRoleId'] = $dbServer->farmRoleId;
$item['serverIndex'] = $dbServer->index;
$item['serverId'] = $dbServer->serverId;
$item['farmName'] = $dbServer->GetFarmObject()->Name;
$item['roleName'] = $dbServer->GetFarmRoleObject()->GetRoleObject()->name;
} catch (Exception $e) {
}
}
$ips[] = $item;
}
$response = $this->buildResponseFromData($ips, array('serverId', 'ipId', 'ip', 'farmId', 'farmRoleId'));
$this->response->data($response);
}
public function getSnapshots($platformName, $snapshotId = null)
{
if (!$platformName) {
throw new Exception("Cloud should be specified");
}
$platform = PlatformFactory::NewPlatform($platformName);
$cs = Scalr_Service_Cloud_Cloudstack::newCloudstack($platform->getConfigVariable(Modules_Platforms_Cloudstack::API_URL, $this->environment), $platform->getConfigVariable(Modules_Platforms_Cloudstack::API_KEY, $this->environment), $platform->getConfigVariable(Modules_Platforms_Cloudstack::SECRET_KEY, $this->environment), $platformName);
$snapshots = $cs->listSnapshots();
$snaps = array();
foreach ($snapshots as $pk => $pv) {
if ($snapshotId && $snapshotId != $pv->id) {
continue;
}
$item = array('snapshotId' => (string) $pv->id, 'type' => $pv->snapshottype, 'volumeId' => $pv->volumeid, 'volumeType' => $pv->volumetype, 'createdAt' => $pv->created, 'intervalType' => $pv->intervaltype, 'state' => $pv->state);
$snaps[] = $item;
}
return $snaps;
}
public function xRemoveAction()
{
$this->request->defineParams(array('ipId' => array('type' => 'json'), 'cloudLocation'));
$platformName = $this->getParam('platform');
if (!$platformName) {
throw new Exception("Cloud should be specified");
}
$platform = PlatformFactory::NewPlatform($platformName);
$networkType = $platform->getConfigVariable(Modules_Platforms_Openstack::NETWORK_TYPE, $this->environment, false);
$openstack = $this->environment->openstack($platformName, $this->getParam('cloudLocation'));
foreach ($this->getParam('ipId') as $ipId) {
if ($networkType == Modules_Platforms_Openstack::NETWORK_TYPE_QUANTUM) {
$openstack->network->floatingIps->delete($ipId);
} else {
$openstack->servers->floatingIps->delete($ipId);
}
}
$this->response->success('Floating IP(s) successfully removed');
}
public static function onCreateBackupResult(Scalr_Messaging_Msg $message, DBServer $dbServer)
{
$dbFarmRole = $dbServer->GetFarmRoleObject();
$dbFarmRole->SetSetting(Scalr_Db_Msr::DATA_BACKUP_LAST_TS, time(), DBFarmRole::TYPE_LCL);
$dbFarmRole->SetSetting(Scalr_Db_Msr::DATA_BACKUP_IS_RUNNING, 0, DBFarmRole::TYPE_LCL);
//$dbFarmRole->SetSetting(Scalr_Db_Msr::DATA_BACKUP_SERVER_ID, "");
if (PlatformFactory::isOpenstack($dbServer->platform)) {
$provider = 'cf';
} else {
switch ($dbServer->platform) {
case SERVER_PLATFORMS::EC2:
$provider = 's3';
break;
case SERVER_PLATFORMS::GCE:
$provider = 'gcs';
break;
default:
$provider = 'unknown';
break;
}
}
$backup = Scalr_Db_Backup::init();
$backup->service = $message->dbType;
$backup->platform = $dbServer->platform;
$backup->provider = $provider;
$backup->envId = $dbServer->envId;
$backup->farmId = $dbServer->farmId;
$backup->cloudLocation = $dbServer->GetCloudLocation();
$backup->status = Scalr_Db_Backup::STATUS_AVAILABLE;
$total = 0;
if (isset($message->backupParts) && is_array($message->backupParts)) {
foreach ($message->backupParts as $item) {
if (is_object($item) && $item->size) {
$backup->addPart(str_replace(array("s3://", "cf://", "gcs://", "swift://"), array("", "", "", ""), $item->path), $item->size);
$total = $total + (int) $item->size;
} else {
$backup->addPart(str_replace(array("s3://", "cf://", "gcs://", "swift://"), array("", "", "", ""), $item), 0);
}
}
}
$backup->size = $total;
$backup->save();
}
/**
*
* @param string $platform (From SERVER_PLATFORMS class)
* @param integer $farmid
* @param integer $farm_roleid
* @param integer $index
* @return void
*/
public function __construct($platform, DBFarmRole $DBFarmRole = null, $index = null, $role_id = null)
{
$this->platform = $platform;
$this->dbFarmRole = $DBFarmRole;
$this->index = $index;
$this->roleId = $role_id === null ? $this->dbFarmRole->RoleID : $role_id;
if ($DBFarmRole) {
$this->envId = $DBFarmRole->GetFarmObject()->EnvID;
}
//Refletcion
$Reflect = new ReflectionClass(DBServer::$platformPropsClasses[$this->platform]);
foreach ($Reflect->getConstants() as $k => $v) {
$this->platformProps[] = $v;
}
if ($DBFarmRole) {
if (PlatformFactory::isOpenstack($this->platform)) {
$this->SetProperties(array(OPENSTACK_SERVER_PROPERTIES::CLOUD_LOCATION => $DBFarmRole->CloudLocation));
} elseif (PlatformFactory::isCloudstack($this->platform)) {
$this->SetProperties(array(CLOUDSTACK_SERVER_PROPERTIES::CLOUD_LOCATION => $DBFarmRole->CloudLocation));
} else {
switch ($this->platform) {
case SERVER_PLATFORMS::NIMBULA:
break;
case SERVER_PLATFORMS::EUCALYPTUS:
$this->SetProperties(array(EUCA_SERVER_PROPERTIES::REGION => $DBFarmRole->CloudLocation));
break;
case SERVER_PLATFORMS::RACKSPACE:
$this->SetProperties(array(RACKSPACE_SERVER_PROPERTIES::DATACENTER => $DBFarmRole->CloudLocation));
break;
case SERVER_PLATFORMS::GCE:
$this->SetProperties(array(GCE_SERVER_PROPERTIES::CLOUD_LOCATION => $DBFarmRole->CloudLocation));
break;
case SERVER_PLATFORMS::EC2:
$this->SetProperties(array(EC2_SERVER_PROPERTIES::REGION => $DBFarmRole->CloudLocation));
break;
}
}
//TODO:
$this->SetProperties(array(SERVER_PROPERTIES::SZR_VESION => '0.20.0'));
} else {
$this->SetProperties(array(SERVER_PROPERTIES::SZR_VESION => '0.20.0'));
}
}
public function xGetMachineTypesAction()
{
$p = PlatformFactory::NewPlatform(SERVER_PLATFORMS::GCE);
$client = new Google_Client();
$client->setApplicationName("Scalr GCE");
$client->setScopes(array('https://www.googleapis.com/auth/compute'));
$key = base64_decode($this->environment->getPlatformConfigValue(Modules_Platforms_GoogleCE::KEY));
$client->setAssertionCredentials(new Google_AssertionCredentials($this->environment->getPlatformConfigValue(Modules_Platforms_GoogleCE::SERVICE_ACCOUNT_NAME), array('https://www.googleapis.com/auth/compute'), $key));
$client->setUseObjects(true);
$client->setClientId($this->environment->getPlatformConfigValue(Modules_Platforms_GoogleCE::CLIENT_ID));
$projectId = $this->environment->getPlatformConfigValue(Modules_Platforms_GoogleCE::PROJECT_ID);
$gceClient = new Google_ComputeService($client);
$data['types'] = array();
$data['dbTypes'] = array();
$types = $gceClient->machineTypes->listMachineTypes($projectId, $this->getParam('cloudLocation'));
foreach ($types->items as $item) {
$isEphemeral = substr($item->name, -2) == '-d';
if (!$isEphemeral) {
$data['types'][] = array('name' => $item->name, 'description' => "{$item->name} ({$item->description})");
}
}
$this->response->data(array('data' => $data));
}
public function onFarmSave(DBFarm $dbFarm, DBFarmRole $dbFarmRole)
{
$vpcId = $dbFarm->GetSetting(DBFarm::SETTING_EC2_VPC_ID);
if (!$vpcId) {
//REMOVE VPC RELATED SETTINGS
return;
}
if ($dbFarmRole->GetSetting(self::ROLE_VPC_ROUTER_CONFIGURED) == 1) {
// ALL OBJECTS ALREADY CONFIGURED
return true;
}
$aws = $dbFarm->GetEnvironmentObject()->aws($dbFarmRole->CloudLocation);
$filter = array(array('name' => SubnetFilterNameType::vpcId(), 'value' => $vpcId), array('name' => SubnetFilterNameType::tagKey(), 'value' => 'scalr-sn-type'), array('name' => SubnetFilterNameType::tagValue(), 'value' => self::INTERNET_ACCESS_FULL));
// Try to find scalr FULL subnet
$subnets = $aws->ec2->subnet->describe(null, $filter);
if ($subnets->count() > 0) {
$subnetId = $subnets->get(0)->subnetId;
}
if (!$subnetId) {
$platform = PlatformFactory::NewPlatform(SERVER_PLATFORMS::EC2);
$subnet = $platform->AllocateNewSubnet($aws->ec2, $vpcId, null);
$subnetId = $subnet->subnetId;
//ADD TAGS
try {
$subnet->createTags(array(array('key' => "scalr-id", 'value' => SCALR_ID), array('key' => "scalr-sn-type", 'value' => self::INTERNET_ACCESS_FULL), array('key' => "Name", 'value' => 'Scalr System Subnet')));
} catch (Exception $e) {
}
$routingTableId = $platform->getRoutingTable(self::INTERNET_ACCESS_FULL, $aws, null, $vpcId);
//Associate Routing table with subnet
$aws->ec2->routeTable->associate($routingTableId, $subnetId);
}
$niId = $dbFarmRole->GetSetting(self::ROLE_VPC_NID);
if (!$niId) {
//Create Network interface
$createNetworkInterfaceRequestData = new CreateNetworkInterfaceRequestData($subnetId);
// Check and create security group
$filter = array(array('name' => SecurityGroupFilterNameType::groupName(), 'value' => array('SCALR-VPC')), array('name' => SecurityGroupFilterNameType::vpcId(), 'value' => $vpcId));
try {
$list = $aws->ec2->securityGroup->describe(null, null, $filter);
if ($list->count() > 0 && $list->get(0)->groupName == 'SCALR-VPC') {
$sgId = $list->get(0)->groupId;
}
} catch (Exception $e) {
throw new Exception("Cannot get list of security groups (1): {$e->getMessage()}");
}
if (!$sgId) {
$sgId = $aws->ec2->securityGroup->create('SCALR-VPC', 'System SG for Scalr VPC integration', $vpcId);
$ipRangeList = new IpRangeList();
$ipRangeList->append(new IpRangeData('0.0.0.0/0'));
$ipRangeListLocal = new IpRangeList();
$ipRangeListLocal->append(new IpRangeData('10.0.0.0/8'));
$aws->ec2->securityGroup->authorizeIngress(array(new IpPermissionData('tcp', 8008, 8013, $ipRangeList), new IpPermissionData('tcp', 80, 80, $ipRangeList), new IpPermissionData('tcp', 443, 443, $ipRangeList), new IpPermissionData('tcp', 0, 65535, $ipRangeListLocal), new IpPermissionData('udp', 0, 65535, $ipRangeListLocal)), $sgId);
}
$createNetworkInterfaceRequestData->setSecurityGroupId(array('groupId' => $sgId));
$networkInterface = $aws->ec2->networkInterface->create($createNetworkInterfaceRequestData);
// Disable sourceDeskCheck
$networkInterface->modifyAttribute(NetworkInterfaceAttributeType::sourceDestCheck(), 0);
$niId = $networkInterface->networkInterfaceId;
$dbFarmRole->SetSetting(self::ROLE_VPC_NID, $niId, DBFarmRole::TYPE_LCL);
try {
$networkInterface->createTags(array(array('key' => "scalr-id", 'value' => SCALR_ID), array('key' => "Name", 'value' => 'Scalr System ENI')));
} catch (Exception $e) {
}
}
// If there is no public IP allocate it and associate with NI
$publicIp = $dbFarmRole->GetSetting(self::ROLE_VPC_IP);
if ($niId && !$publicIp) {
$address = $aws->ec2->address->allocate('vpc');
$publicIp = $address->publicIp;
$dbFarmRole->SetSetting(self::ROLE_VPC_IP, $publicIp, DBFarmRole::TYPE_LCL);
$dbFarmRole->SetSetting(self::ROLE_VPC_AID, $address->allocationId, DBFarmRole::TYPE_LCL);
$associateAddressRequestData = new AssociateAddressRequestData();
$associateAddressRequestData->networkInterfaceId = $niId;
$associateAddressRequestData->allocationId = $address->allocationId;
//Associate PublicIP with NetworkInterface
$aws->ec2->address->associate($associateAddressRequestData);
}
$dbFarmRole->SetSetting(self::ROLE_VPC_ROUTER_CONFIGURED, 1, DBFarmRole::TYPE_LCL);
}
/**
*
* @param ServerCreateInfo $ServerCreateInfo
* @return DBServer
*/
public static function LaunchServer(ServerCreateInfo $ServerCreateInfo = null, DBServer $DBServer = null, $delayed = false)
{
$db = Core::GetDBInstance();
if (!$DBServer && $ServerCreateInfo) {
$ServerCreateInfo->SetProperties(array(SERVER_PROPERTIES::SZR_KEY => Scalr::GenerateRandomKey(40), SERVER_PROPERTIES::SZR_KEY_TYPE => SZR_KEY_TYPE::ONE_TIME));
$DBServer = DBServer::Create($ServerCreateInfo, false, true);
} elseif (!$DBServer && !$ServerCreateInfo) {
// incorrect arguments
Logger::getLogger(LOG_CATEGORY::FARM)->error(sprintf("Cannot create server"));
return null;
}
if ($delayed) {
$DBServer->status = SERVER_STATUS::PENDING_LAUNCH;
$DBServer->Save();
return $DBServer;
}
try {
$account = Scalr_Account::init()->loadById($DBServer->clientId);
$account->validateLimit(Scalr_Limits::ACCOUNT_SERVERS, 1);
PlatformFactory::NewPlatform($DBServer->platform)->LaunchServer($DBServer);
$DBServer->status = SERVER_STATUS::PENDING;
$DBServer->Save();
} catch (Exception $e) {
Logger::getLogger(LOG_CATEGORY::FARM)->error(new FarmLogMessage($DBServer->farmId, sprintf("Cannot launch server on '%s' platform: %s", $DBServer->platform, $e->getMessage())));
$DBServer->status = SERVER_STATUS::PENDING_LAUNCH;
$DBServer->SetProperty(SERVER_PROPERTIES::LAUNCH_ERROR, $e->getMessage());
$DBServer->Save();
}
if ($DBServer->status == SERVER_STATUS::PENDING) {
Scalr::FireEvent($DBServer->farmId, new BeforeInstanceLaunchEvent($DBServer));
$DBServer->SetProperty(SERVER_PROPERTIES::LAUNCH_ERROR, "");
$db->Execute("UPDATE servers_history SET\r\n\t\t\t\t\t`dtlaunched` = NOW(),\r\n\t\t\t\t\t`cloud_server_id` = ?,\r\n\t\t\t\t\t`type` = ?\r\n\t\t\t\t\tWHERE server_id = ?\r\n\t\t\t\t", array($DBServer->GetCloudServerID(), $DBServer->GetProperty(EC2_SERVER_PROPERTIES::INSTANCE_TYPE), $DBServer->serverId));
}
return $DBServer;
}
function handleWork($serverId)
{
try {
$dbserver = DBServer::LoadByID($serverId);
if ($dbserver->farmId) {
if ($dbserver->GetFarmObject()->Status == FARM_STATUS::TERMINATED) {
throw new ServerNotFoundException("");
}
}
} catch (Exception $e) {
$this->db->Execute("DELETE FROM messages WHERE server_id=? AND `type`='in'", array($serverId));
return;
}
$rs = $this->db->Execute("SELECT * FROM messages \n \t\tWHERE server_id = ? AND type = ? AND status = ? \n \t\tORDER BY id ASC", array($serverId, "in", MESSAGE_STATUS::PENDING));
while ($row = $rs->FetchRow()) {
try {
$message = $this->serializer->unserialize($row["message"]);
$event = null;
// Update scalarizr package version
if ($message->meta[Scalr_Messaging_MsgMeta::SZR_VERSION]) {
$dbserver->SetProperty(SERVER_PROPERTIES::SZR_VESION, $message->meta[Scalr_Messaging_MsgMeta::SZR_VERSION]);
}
if ($message->meta[Scalr_Messaging_MsgMeta::SZR_UPD_CLIENT_VERSION]) {
$dbserver->SetProperty(SERVER_PROPERTIES::SZR_UPD_CLIENT_VERSION, $message->meta[Scalr_Messaging_MsgMeta::SZR_UPD_CLIENT_VERSION]);
}
try {
if ($message instanceof Scalr_Messaging_Msg_OperationResult) {
$this->db->Execute("UPDATE server_operations SET `status` = ? WHERE id = ?", array($message->status, $message->id));
if ($message->status == 'error') {
if ($message->name == 'Initialization') {
$dbserver->SetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED, 1);
}
if ($message->error) {
$msg = $message->error->message;
$trace = $message->error->trace;
$handler = $message->error->handler;
}
$this->db->Execute("INSERT INTO server_operation_progress SET \n\t\t\t\t\t\t\t\t\t`operation_id` = ?,\n\t\t\t\t\t\t\t\t\t`timestamp` = ?,\n\t\t\t\t\t\t\t\t\t`phase` = ?,\n\t\t\t\t\t\t\t\t\t`step` = ?,\n\t\t\t\t\t\t\t\t\t`status` = ?,\n\t\t\t\t\t\t\t\t\t`message`= ?,\n\t\t\t\t\t\t\t\t\t`trace` = ?,\n\t\t\t\t\t\t\t\t\t`handler` = ?,\n\t\t\t\t\t\t\t\t\t`progress` = ?,\n\t\t\t\t\t\t\t\t\t`stepno` = ? \n\t\t\t\t\t\t\t\t\tON DUPLICATE KEY UPDATE status = ?, progress = ?, trace = ?, handler = ?, message = ?\n\t\t\t\t\t\t\t\t", array($message->id, $message->getTimestamp(), $message->phase, $message->step, $message->status, $msg, $trace, $handler, $message->progress, $message->stepno, $message->status, $message->progress, $trace, $handler, $msg));
}
} elseif ($message instanceof Scalr_Messaging_Msg_Win_HostDown) {
$status = PlatformFactory::NewPlatform($dbserver->platform)->GetServerRealStatus($dbserver);
if ($status->isRunning()) {
$event = new RebootBeginEvent($dbserver);
} else {
$event = new HostDownEvent($dbserver);
}
} elseif ($message instanceof Scalr_Messaging_Msg_Win_PrepareBundleResult) {
try {
$bundleTask = BundleTask::LoadById($message->bundleTaskId);
} catch (Exception $e) {
}
if ($bundleTask) {
if ($message->status == 'ok') {
$metaData = array('szr_version' => $message->meta[Scalr_Messaging_MsgMeta::SZR_VERSION], 'os' => $message->os, 'software' => $message->software);
$bundleTask->setMetaData($metaData);
$bundleTask->Save();
PlatformFactory::NewPlatform($bundleTask->platform)->CreateServerSnapshot($bundleTask);
} else {
$bundleTask->SnapshotCreationFailed("PrepareBundle procedure failed: {$message->lastError}");
}
}
} elseif ($message instanceof Scalr_Messaging_Msg_DeployResult) {
try {
$deploymentTask = Scalr_Model::init(Scalr_Model::DM_DEPLOYMENT_TASK)->loadById($message->deployTaskId);
} catch (Exception $e) {
}
if ($deploymentTask) {
if ($message->status == 'error') {
$deploymentTask->status = Scalr_Dm_DeploymentTask::STATUS_FAILED;
$deploymentTask->lastError = $message->lastError;
} else {
$deploymentTask->status = Scalr_Dm_DeploymentTask::STATUS_DEPLOYED;
$deploymentTask->dtDeployed = date("Y-m-d H:i:s");
}
$deploymentTask->save();
}
} elseif ($message instanceof Scalr_Messaging_Msg_Hello) {
$event = $this->onHello($message, $dbserver);
} elseif ($message instanceof Scalr_Messaging_Msg_MongoDb) {
try {
$dbFarmRole = $dbserver->GetFarmRoleObject();
} catch (Exception $e) {
}
if ($dbFarmRole instanceof DBFarmRole) {
foreach (Scalr_Role_Behavior::getListForFarmRole($dbFarmRole) as $behavior) {
$behavior->handleMessage($message, $dbserver);
}
}
} elseif ($message instanceof Scalr_Messaging_Msg_DbMsr) {
try {
$dbFarmRole = $dbserver->GetFarmRoleObject();
} catch (Exception $e) {
}
if ($dbFarmRole instanceof DBFarmRole) {
foreach (Scalr_Role_Behavior::getListForFarmRole($dbFarmRole) as $behavior) {
$behavior->handleMessage($message, $dbserver);
}
}
} elseif ($message instanceof Scalr_Messaging_Msg_HostInit) {
$event = $this->onHostInit($message, $dbserver);
} elseif ($message instanceof Scalr_Messaging_Msg_HostUp) {
$event = $this->onHostUp($message, $dbserver);
} elseif ($message instanceof Scalr_Messaging_Msg_HostDown) {
$isMoving = false;
if ($dbserver->platform == SERVER_PLATFORMS::RACKSPACE) {
$p = PlatformFactory::NewPlatform($dbserver->platform);
$status = $p->GetServerRealStatus($dbserver)->getName();
if (stristr($status, 'MOVE') || stristr($status, 'REBOOT')) {
$this->logger->error(new FarmLogMessage($dbserver->farmId, "Rackspace server is in MOVING state. Ignoring HostDown message."));
$isMoving = true;
}
}
if (!$isMoving) {
$event = new HostDownEvent($dbserver);
}
} elseif ($message instanceof Scalr_Messaging_Msg_RebootStart) {
$event = new RebootBeginEvent($dbserver);
} elseif ($message instanceof Scalr_Messaging_Msg_RebootFinish) {
$event = new RebootCompleteEvent($dbserver);
} elseif ($message instanceof Scalr_Messaging_Msg_BeforeHostUp) {
$event = new BeforeHostUpEvent($dbserver);
} elseif ($message instanceof Scalr_Messaging_Msg_BlockDeviceAttached) {
if ($dbserver->platform == SERVER_PLATFORMS::EC2) {
$ec2Client = Scalr_Service_Cloud_Aws::newEc2($dbserver->GetProperty(EC2_SERVER_PROPERTIES::REGION), $dbserver->GetEnvironmentObject()->getPlatformConfigValue(Modules_Platforms_Ec2::PRIVATE_KEY), $dbserver->GetEnvironmentObject()->getPlatformConfigValue(Modules_Platforms_Ec2::CERTIFICATE));
$instanceId = $dbserver->GetProperty(EC2_SERVER_PROPERTIES::INSTANCE_ID);
$volumes = $ec2Client->DescribeVolumes()->volumeSet->item;
if (!is_array($volumes)) {
$volumes = array($volumes);
}
foreach ($volumes as $volume) {
if ($volume->status == AMAZON_EBS_STATE::IN_USE && $volume->attachmentSet->item->instanceId == $instanceId && $volume->attachmentSet->item->device == $message->deviceName) {
$message->volumeId = $volume->volumeId;
}
}
}
$event = new EBSVolumeAttachedEvent($dbserver, $message->deviceName, $message->volumeId);
} elseif ($message instanceof Scalr_Messaging_Msg_BlockDeviceMounted) {
// Single volume
$ebsinfo = $this->db->GetRow("SELECT * FROM ec2_ebs WHERE volume_id=?", array($message->volumeId));
if ($ebsinfo) {
$this->db->Execute("UPDATE ec2_ebs SET mount_status=?, isfsexist='1' WHERE id=?", array(EC2_EBS_MOUNT_STATUS::MOUNTED, $ebsinfo['id']));
}
$event = new EBSVolumeMountedEvent($dbserver, $message->mountpoint, $message->volumeId, $message->deviceName);
} elseif ($message instanceof Scalr_Messaging_Msg_RebundleResult) {
if ($message->status == Scalr_Messaging_Msg_RebundleResult::STATUS_OK) {
$metaData = array('szr_version' => $message->meta[Scalr_Messaging_MsgMeta::SZR_VERSION], 'dist' => $message->dist, 'os' => $message->os, 'software' => $message->software);
if ($dbserver->platform == SERVER_PLATFORMS::EC2) {
if ($message->aws) {
if ($message->aws->root - device - type == 'ebs') {
$tags[] = ROLE_TAGS::EC2_EBS;
}
if ($message->aws->virtualization - type == 'hvm') {
$tags[] = ROLE_TAGS::EC2_HVM;
}
} else {
$ec2Client = Scalr_Service_Cloud_Aws::newEc2($dbserver->GetProperty(EC2_SERVER_PROPERTIES::REGION), $dbserver->GetEnvironmentObject()->getPlatformConfigValue(Modules_Platforms_Ec2::PRIVATE_KEY), $dbserver->GetEnvironmentObject()->getPlatformConfigValue(Modules_Platforms_Ec2::CERTIFICATE));
try {
$DescribeImagesType = new DescribeImagesType(null, array(), null);
$DescribeImagesType->imagesSet = new stdClass();
$DescribeImagesType->imagesSet->item = array();
$DescribeImagesType->imagesSet->item[] = array('imageId' => $dbserver->GetProperty(EC2_SERVER_PROPERTIES::AMIID));
$info = $ec2Client->DescribeImages($DescribeImagesType);
if ($info->imagesSet->item->rootDeviceType == 'ebs') {
$tags[] = ROLE_TAGS::EC2_EBS;
} else {
try {
$bundleTask = BundleTask::LoadById($message->bundleTaskId);
if ($bundleTask->bundleType == SERVER_SNAPSHOT_CREATION_TYPE::EC2_EBS) {
$tags[] = ROLE_TAGS::EC2_EBS;
}
} catch (Exception $e) {
}
}
if ($info->imagesSet->item->virtualizationType == 'hvm') {
$tags[] = ROLE_TAGS::EC2_HVM;
}
} catch (Exception $e) {
$metaData['tagsError'] = $e->getMessage();
}
}
} elseif ($dbserver->platform == SERVER_PLATFORMS::NIMBULA) {
$metaData['init_root_user'] = $message->sshUser;
$metaData['init_root_pass'] = $message->sshPassword;
}
$metaData['tags'] = $tags;
$event = new RebundleCompleteEvent($dbserver, $message->snapshotId, $message->bundleTaskId, $metaData);
} else {
if ($message->status == Scalr_Messaging_Msg_RebundleResult::STATUS_FAILED) {
$event = new RebundleFailedEvent($dbserver, $message->bundleTaskId, $message->lastError);
}
}
} elseif ($message instanceof Scalr_Messaging_Msg_Mysql_CreateDataBundleResult) {
if ($message->status == "ok") {
$event = new MysqlBackupCompleteEvent($dbserver, MYSQL_BACKUP_TYPE::BUNDLE, array('snapshotConfig' => $message->snapshotConfig, 'logFile' => $message->logFile, 'logPos' => $message->logPos, 'dataBundleSize' => $message->dataBundleSize, 'snapshotId' => $message->snapshotId));
} else {
$event = new MysqlBackupFailEvent($dbserver, MYSQL_BACKUP_TYPE::BUNDLE);
$event->lastError = $message->lastError;
}
} elseif ($message instanceof Scalr_Messaging_Msg_Mysql_CreateBackupResult) {
if ($message->status == "ok") {
$event = new MysqlBackupCompleteEvent($dbserver, MYSQL_BACKUP_TYPE::DUMP);
} else {
$event = new MysqlBackupFailEvent($dbserver, MYSQL_BACKUP_TYPE::DUMP);
$event->lastError = $message->lastError;
}
} elseif ($message instanceof Scalr_Messaging_Msg_Mysql_PromoteToMasterResult) {
$event = $this->onMysql_PromoteToMasterResult($message, $dbserver);
} elseif ($message instanceof Scalr_Messaging_Msg_Mysql_CreatePmaUserResult) {
$farmRole = DBFarmRole::LoadByID($message->farmRoleId);
if ($message->status == "ok") {
$farmRole->SetSetting(DbFarmRole::SETTING_MYSQL_PMA_USER, $message->pmaUser);
$farmRole->SetSetting(DbFarmRole::SETTING_MYSQL_PMA_PASS, $message->pmaPassword);
} else {
$farmRole->SetSetting(DBFarmRole::SETTING_MYSQL_PMA_REQUEST_TIME, "");
$farmRole->SetSetting(DBFarmRole::SETTING_MYSQL_PMA_REQUEST_ERROR, $message->lastError);
}
} elseif ($message instanceof Scalr_Messaging_Msg_RabbitMq_SetupControlPanelResult) {
$farmRole = $dbserver->GetFarmRoleObject();
if ($message->status == "ok") {
$farmRole->SetSetting(Scalr_Role_Behavior_RabbitMQ::ROLE_CP_SERVER_ID, $dbserver->serverId);
$farmRole->SetSetting(Scalr_Role_Behavior_RabbitMQ::ROLE_CP_URL, $message->cpanelUrl);
$farmRole->SetSetting(Scalr_Role_Behavior_RabbitMQ::ROLE_CP_REQUEST_TIME, "");
} else {
$farmRole->SetSetting(Scalr_Role_Behavior_RabbitMQ::ROLE_CP_SERVER_ID, "");
$farmRole->SetSetting(Scalr_Role_Behavior_RabbitMQ::ROLE_CP_REQUEST_TIME, "");
$farmRole->SetSetting(Scalr_Role_Behavior_RabbitMQ::ROLE_CP_ERROR_MSG, $message->lastError);
}
} elseif ($message instanceof Scalr_Messaging_Msg_AmiScriptsMigrationResult) {
try {
//Open security group:
if ($dbserver->platform == SERVER_PLATFORMS::EC2) {
$info = PlatformFactory::NewPlatform($dbserver->platform)->GetServerExtendedInformation($dbserver);
$sg = explode(", ", $info['Security groups']);
foreach ($sg as $sgroup) {
if ($sgroup != 'default') {
$ipPermissionSet = new IpPermissionSetType();
$group_rules = array(array('rule' => 'tcp:8013:8013:0.0.0.0/0'), array('rule' => 'udp:8014:8014:0.0.0.0/0'));
foreach ($group_rules as $rule) {
$group_rule = explode(":", $rule["rule"]);
$ipPermissionSet->AddItem($group_rule[0], $group_rule[1], $group_rule[2], null, array($group_rule[3]));
}
$ec2Client = Scalr_Service_Cloud_Aws::newEc2($dbserver->GetProperty(EC2_SERVER_PROPERTIES::REGION), $dbserver->GetEnvironmentObject()->getPlatformConfigValue(Modules_Platforms_Ec2::PRIVATE_KEY), $dbserver->GetEnvironmentObject()->getPlatformConfigValue(Modules_Platforms_Ec2::CERTIFICATE));
// Create security group
$ec2Client->AuthorizeSecurityGroupIngress($dbserver->GetEnvironmentObject()->getPlatformConfigValue(Modules_Platforms_Ec2::ACCOUNT_ID), $sgroup, $ipPermissionSet);
break;
}
}
}
} catch (Exception $e) {
$this->logger->fatal($e->getMessage());
}
$dbserver->SetProperty(SERVER_PROPERTIES::SZR_SNMP_PORT, 8014);
$dbserver->SetProperty(SERVER_PROPERTIES::SZR_VESION, "0.7.217");
if ($message->mysql) {
$event = $this->onHostUp($message, $dbserver, true);
}
}
$handle_status = MESSAGE_STATUS::HANDLED;
} catch (Exception $e) {
$handle_status = MESSAGE_STATUS::FAILED;
$this->logger->error(sprintf("Cannot handle message '%s' (message_id: %s) " . "from server '%s' (server_id: %s). %s", $message->getName(), $message->messageId, $dbserver->remoteIp ? $dbserver->remoteIp : '*no-ip*', $dbserver->serverId, $e->getMessage() . "({$e->getFile()}:{$e->getLine()})"));
}
$this->db->Execute("UPDATE messages SET status = ? WHERE messageid = ?", array($handle_status, $message->messageId));
if ($event) {
Scalr::FireEvent($dbserver->farmId, $event);
}
} catch (Exception $e) {
$this->logger->error($e->getMessage(), $e);
}
}
}
/**
* Delete role from farm
* @return void
*/
public function Delete()
{
foreach ($this->GetServersByFilter() as $DBServer) {
if ($DBServer->status != SERVER_STATUS::TERMINATED) {
try {
PlatformFactory::NewPlatform($DBServer->platform)->TerminateServer($DBServer);
$this->DB->Execute("UPDATE servers_history SET\r\n\t\t\t\t\t\t\tdtterminated\t= NOW(),\r\n\t\t\t\t\t\t\tterminate_reason\t= ?\r\n\t\t\t\t\t\t\tWHERE server_id = ?\r\n\t\t\t\t\t\t", array(sprintf("Role removed from farm"), $DBServer->serverId));
} catch (Exception $e) {
}
$DBServer->status = SERVER_STATUS::TERMINATED;
if (defined("SCALR_SERVER_TZ")) {
$tz = date_default_timezone_get();
date_default_timezone_set(SCALR_SERVER_TZ);
}
$DBServer->dateShutdownScheduled = date("Y-m-d H:i:s");
if ($tz) {
date_default_timezone_set($tz);
}
$event = new HostDownEvent($DBServer);
Scalr::FireEvent($DBServer->farmId, $event);
$DBServer->Save();
}
}
//
$this->DB->Execute("DELETE FROM farm_roles WHERE id=?", array($this->ID));
// Clear farm role options & scripts
$this->DB->Execute("DELETE FROM farm_role_options WHERE farm_roleid=?", array($this->ID));
$this->DB->Execute("DELETE FROM farm_role_scripts WHERE farm_roleid=?", array($this->ID));
$this->DB->Execute("DELETE FROM farm_role_service_config_presets WHERE farm_roleid=?", array($this->ID));
$this->DB->Execute("DELETE FROM farm_role_scaling_metrics WHERE farm_roleid=?", array($this->ID));
$this->DB->Execute("DELETE FROM farm_role_scaling_times WHERE farm_roleid=?", array($this->ID));
$this->DB->Execute("DELETE FROM farm_role_service_config_presets WHERE farm_roleid=?", array($this->ID));
$this->DB->Execute("DELETE FROM farm_role_settings WHERE farm_roleid=?", array($this->ID));
$this->DB->Execute("DELETE FROM ec2_ebs WHERE farm_roleid=?", array($this->ID));
$this->DB->Execute("DELETE FROM elastic_ips WHERE farm_roleid=?", array($this->ID));
$this->DB->Execute("DELETE FROM storage_volumes WHERE farm_roleid=?", array($this->ID));
// Clear apache vhosts and update DNS zones
$this->DB->Execute("UPDATE apache_vhosts SET farm_roleid='0' WHERE farm_roleid=?", array($this->ID));
$this->DB->Execute("UPDATE dns_zones SET farm_roleid='0' WHERE farm_roleid=?", array($this->ID));
// Clear scheduler tasks
$this->DB->Execute("DELETE FROM scheduler WHERE target_id = ? AND target_type = ?", array($this->ID, Scalr_SchedulerTask::TARGET_ROLE));
$this->DB->Execute("DELETE FROM scheduler WHERE target_id LIKE '" . $this->ID . ":%' AND target_type = ?", array(Scalr_SchedulerTask::TARGET_INSTANCE));
}
public function StartThread($bundle_task_info)
{
$db = Core::GetDBInstance();
// Reconfigure observers;
Scalr::ReconfigureObservers();
$BundleTask = BundleTask::LoadById($bundle_task_info['id']);
try {
$DBServer = DBServer::LoadByID($BundleTask->serverId);
} catch (ServerNotFoundException $e) {
if (!$BundleTask->snapshotId) {
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::FAILED;
$BundleTask->setDate('finished');
$BundleTask->failureReason = sprintf(_("Server '%s' was terminated during snapshot creation process"), $BundleTask->serverId);
$BundleTask->Save();
return;
}
} catch (Exception $e) {
//$this->Logger->error($e->getMessage());
}
switch ($BundleTask->status) {
case SERVER_SNAPSHOT_CREATION_STATUS::STARING_SERVER:
case SERVER_SNAPSHOT_CREATION_STATUS::PREPARING_ENV:
case SERVER_SNAPSHOT_CREATION_STATUS::INTALLING_SOFTWARE:
if (!PlatformFactory::NewPlatform($DBServer->platform)->IsServerExists($DBServer)) {
$DBServer->status = SERVER_STATUS::TERMINATED;
$DBServer->save();
$BundleTask->SnapshotCreationFailed("Server was terminated and no longer available in cloud.");
exit;
}
// IF server is in pensing state
$status = PlatformFactory::NewPlatform($DBServer->platform)->GetServerRealStatus($DBServer);
$BundleTask->Log(sprintf(_("Server status: %s"), $status->getName()));
if ($status->isPending()) {
$BundleTask->Log(sprintf(_("Waiting for running state."), $status->getName()));
exit;
} elseif ($status->isTerminated()) {
$DBServer->status = SERVER_STATUS::TERMINATED;
$DBServer->save();
$BundleTask->SnapshotCreationFailed("Server was terminated and no longer available in cloud.");
exit;
}
break;
}
switch ($BundleTask->status) {
case SERVER_SNAPSHOT_CREATION_STATUS::STARING_SERVER:
$ips = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
$DBServer->remoteIp = $ips['remoteIp'];
$DBServer->localIp = $ips['locateIp'];
$DBServer->save();
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::PREPARING_ENV;
$BundleTask->save();
$BundleTask->Log(sprintf(_("Bundle task status: %s"), $BundleTask->status));
break;
case SERVER_SNAPSHOT_CREATION_STATUS::PREPARING_ENV:
$BundleTask->Log(sprintf(_("Initializing SSH2 session to the server")));
try {
$ssh2Client = $DBServer->GetSsh2Client();
$ssh2Client->connect($DBServer->remoteIp, 22);
} catch (Exception $e) {
$BundleTask->Log(sprintf(_("Scalr unable to establish SSH connection with server on %:%. Error: %s"), $DBServer->remoteIp, 22, $e->getMessage()));
//TODO: Set status of bundle log to failed
exit;
}
//Prepare script
$BundleTask->Log(sprintf(_("Uploading builder scripts...")));
$behaviors = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_BEHAVIOR);
try {
$options = array('server-id' => $DBServer->serverId, 'role-name' => $BundleTask->roleName, 'crypto-key' => $DBServer->GetProperty(SERVER_PROPERTIES::SZR_KEY), 'platform' => $DBServer->platform, 'behaviour' => trim(str_replace("base", "", $behaviors)), 'queryenv-url' => CONFIG::$HTTP_PROTO . "://" . CONFIG::$EVENTHANDLER_URL . "/query-env", 'messaging-p2p.producer-url' => CONFIG::$HTTP_PROTO . "://" . CONFIG::$EVENTHANDLER_URL . "/messaging");
$command = 'scalarizr --import -y';
foreach ($options as $k => $v) {
$command .= sprintf(' -o %s=%s', $k, $v);
}
if ($DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_MYSQL_SERVER_TYPE) == 'percona') {
$recipes = 'mysql=percona';
} else {
$recipes = '';
}
$scalarizrBranch = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_DEV_SCALARIZR_BRANCH);
$scriptContents = @file_get_contents(APPPATH . "/templates/services/role_builder/chef_import.tpl");
$scriptContents = str_replace(array("%PLATFORM%", "%BEHAVIOURS%", "%SZR_IMPORT_STRING%", "%DEV%", "%SCALARIZR_BRANCH%", "%RECIPES%", "%BUILD_ONLY%", "%CHEF_SERVER_URL%", "%CHEF_VALIDATOR_NAME%", "%CHEF_VALIDATOR_KEY%", "%CHEF_ENVIRONMENT%", "%CHEF_ROLE%", "%CHEF_NODE_NAME%", "\r\n"), array($DBServer->platform, trim(str_replace("base", "", str_replace(",", " ", $behaviors))), $command, $scalarizrBranch ? '1' : '0', $scalarizrBranch, $recipes, '0', '', '', '', '', '', '', "\n"), $scriptContents);
if (!$ssh2Client->sendFile('/tmp/scalr-builder.sh', $scriptContents, "w+", false)) {
throw new Exception("Cannot upload script");
}
$BundleTask->Log(sprintf(_("Uploading chef recipes...")));
if (!$ssh2Client->sendFile('/tmp/recipes.tar.gz', APPPATH . '/www/storage/chef/recipes.tar.gz')) {
throw new Exception("Cannot upload chef recipes");
}
} catch (Exception $e) {
$BundleTask->Log(sprintf(_("Scripts upload failed: %s"), $e->getMessage()));
//TODO: Set status of bundle log to failed
exit;
}
$BundleTask->Log("Launching role builder routines on server");
$ssh2Client->exec("chmod 0777 /tmp/scalr-builder.sh");
$ssh2Client->exec("setsid /tmp/scalr-builder.sh > /var/log/role-builder-output.log 2>&1 &");
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::INTALLING_SOFTWARE;
$BundleTask->save();
break;
case SERVER_SNAPSHOT_CREATION_STATUS::INTALLING_SOFTWARE:
try {
$ssh2Client = $DBServer->GetSsh2Client();
$ssh2Client->connect($DBServer->remoteIp, 22);
} catch (Exception $e) {
$BundleTask->Log(sprintf(_("Scalr unable to establish SSH connection with server on %:%. Error: %s"), $DBServer->remoteIp, 22, $e->getMessage()));
//TODO: Set status of bundle log to failed
exit;
}
$log = $ssh2Client->getFile('/var/log/role-builder-output.log');
$log_lines = explode("\r\n", $log);
$last_msg = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_IMPORTING_LAST_LOG_MESSAGE);
while ($msg = trim(array_shift($log_lines))) {
if (substr($msg, -1, 1) != ']') {
continue;
}
if ($last_msg) {
if ($msg != $last_msg) {
continue;
} elseif ($msg == $last_msg) {
$last_msg = null;
continue;
}
}
if (stristr($msg, '[ Failed ]')) {
$stepLog = $ssh2Client->getFile('/var/log/role-builder-step.log');
$BundleTask->Log(sprintf("role-builder-step.log: %s", $stepLog));
$BundleTask->SnapshotCreationFailed($msg);
/** Terminate server **/
PlatformFactory::NewPlatform($DBServer->platform)->TerminateServer($DBServer);
$DBServer->status = SERVER_STATUS::PENDING_TERMINATE;
$DBServer->save();
$BundleTask->Log(sprintf("Temporary server '%s' (%s) has been terminated", $DBServer->serverId, $DBServer->GetProperty(EC2_SERVER_PROPERTIES::INSTANCE_ID)));
} else {
$BundleTask->Log($msg);
$DBServer->SetProperty(SERVER_PROPERTIES::SZR_IMPORTING_LAST_LOG_MESSAGE, $msg);
}
}
//Read /var/log/role-builder-output.log
break;
case SERVER_SNAPSHOT_CREATION_STATUS::PENDING:
try {
$platformModule = PlatformFactory::NewPlatform($BundleTask->platform);
$platformModule->CreateServerSnapshot($BundleTask);
} catch (Exception $e) {
Logger::getLogger(LOG_CATEGORY::BUNDLE)->error($e->getMessage());
$BundleTask->SnapshotCreationFailed($e->getMessage());
}
break;
case SERVER_SNAPSHOT_CREATION_STATUS::PREPARING:
$addedTime = strtotime($BundleTask->dateAdded);
if ($addedTime + 3600 < time()) {
$BundleTask->SnapshotCreationFailed("Server didn't send PrepareBundleResult message in time.");
}
break;
case SERVER_SNAPSHOT_CREATION_STATUS::IN_PROGRESS:
PlatformFactory::NewPlatform($BundleTask->platform)->CheckServerSnapshotStatus($BundleTask);
break;
case SERVER_SNAPSHOT_CREATION_STATUS::REPLACING_SERVERS:
$r_farm_roles = array();
$BundleTask->Log(sprintf("Bundle task replacement type: %s", $BundleTask->replaceType));
try {
$DBFarm = DBFarm::LoadByID($BundleTask->farmId);
} catch (Exception $e) {
if (stristr($e->getMessage(), "not found in database")) {
$BundleTask->SnapshotCreationFailed("Farm was removed before task was finished");
}
return;
}
if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_FARM) {
try {
$r_farm_roles[] = $DBFarm->GetFarmRoleByRoleID($BundleTask->prototypeRoleId);
} catch (Exception $e) {
}
} elseif ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_ALL) {
$farm_roles = $db->GetAll("SELECT id FROM farm_roles WHERE role_id=? AND new_role_id=? AND farmid IN (SELECT id FROM farms WHERE env_id=?)", array($BundleTask->prototypeRoleId, $BundleTask->roleId, $BundleTask->envId));
foreach ($farm_roles as $farm_role) {
try {
$r_farm_roles[] = DBFarmRole::LoadByID($farm_role['id']);
} catch (Exception $e) {
}
}
}
$update_farm_dns_zones = array();
$completed_roles = 0;
foreach ($r_farm_roles as $DBFarmRole) {
if ($DBFarmRole->CloudLocation != $BundleTask->cloudLocation) {
$BundleTask->Log(sprintf("Role '%s' (ID: %s), farm '%s' (ID: %s) using the same role but in abother cloud location. Skiping it.", $DBFarmRole->GetRoleObject()->name, $DBFarmRole->ID, $DBFarmRole->GetFarmObject()->Name, $DBFarmRole->FarmID));
$completed_roles++;
} else {
$servers = $db->GetAll("SELECT server_id FROM servers WHERE farm_roleid = ? AND role_id=? AND status NOT IN (?,?)", array($DBFarmRole->ID, $DBFarmRole->RoleID, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE));
$BundleTask->Log(sprintf("Found %s servers that need to be replaced with new ones. Role '%s' (ID: %s), farm '%s' (ID: %s)", count($servers), $DBFarmRole->GetRoleObject()->name, $DBFarmRole->ID, $DBFarmRole->GetFarmObject()->Name, $DBFarmRole->FarmID));
if (count($servers) == 0) {
$DBFarmRole->RoleID = $DBFarmRole->NewRoleID;
$DBFarmRole->NewRoleID = null;
$DBFarmRole->Save();
$update_farm_dns_zones[$DBFarmRole->FarmID] = 1;
$completed_roles++;
} else {
$metaData = $BundleTask->getSnapshotDetails();
foreach ($servers as $server) {
try {
$DBServer = DBServer::LoadByID($server['server_id']);
} catch (Exception $e) {
//TODO:
continue;
}
if ($DBServer->serverId == $BundleTask->serverId || $metaData['noServersReplace']) {
$DBServer->roleId = $BundleTask->roleId;
$DBServer->Save();
if ($metaData['noServersReplace']) {
$BundleTask->Log(sprintf("'Do not replace servers' option was checked. Server '%s' won't be replaced to new image.", $DBServer->serverId));
} else {
$BundleTask->Log(sprintf("Server '%s', on which snapshot has been taken, already has all modifications. No need to replace it.", $DBServer->serverId));
}
if ($DBServer->GetFarmObject()->Status == FARM_STATUS::SYNCHRONIZING) {
PlatformFactory::NewPlatform($DBServer->platform)->TerminateServer($DBServer);
$db->Execute("UPDATE servers_history SET\r\n\t\t\t\t\t\t\t\t\t\t\t\tdtterminated\t= NOW(),\r\n\t\t\t\t\t\t\t\t\t\t\t\tterminate_reason\t= ?\r\n\t\t\t\t\t\t\t\t\t\t\t\tWHERE server_id = ?\r\n\t\t\t\t\t\t\t\t\t\t\t", array(sprintf("Farm was in 'Synchronizing' state. Server terminated when bundling was completed. Bundle task #%s", $BundleTask->id), $DBServer->serverId));
}
} else {
if (!$db->GetOne("SELECT server_id FROM servers WHERE replace_server_id=? AND status NOT IN (?,?)", array($DBServer->serverId, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE))) {
$ServerCreateInfo = new ServerCreateInfo($DBFarmRole->Platform, $DBFarmRole, $DBServer->index, $DBFarmRole->NewRoleID);
$nDBServer = Scalr::LaunchServer($ServerCreateInfo);
$nDBServer->replaceServerID = $DBServer->serverId;
$nDBServer->Save();
$BundleTask->Log(sprintf(_("Started new server %s to replace server %s"), $nDBServer->serverId, $DBServer->serverId));
}
}
// if serverid != bundletask->serverID
}
// foreach server
}
// count($servers)
}
}
if ($completed_roles == count($r_farm_roles)) {
$BundleTask->Log(sprintf(_("No servers with old role. Replacement complete. Bundle task complete."), SERVER_REPLACEMENT_TYPE::NO_REPLACE, SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS));
$BundleTask->setDate('finished');
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS;
$BundleTask->Save();
}
try {
if (count($update_farm_dns_zones) != 0) {
foreach ($update_farm_dns_zones as $farm_id => $v) {
$dnsZones = DBDNSZone::loadByFarmId($farm_id);
foreach ($dnsZones as $dnsZone) {
if ($dnsZone->status != DNS_ZONE_STATUS::INACTIVE && $dnsZone->status != DNS_ZONE_STATUS::PENDING_DELETE) {
$dnsZone->updateSystemRecords();
$dnsZone->save();
}
}
}
}
} catch (Exception $e) {
$this->Logger->fatal("DNS ZONE: {$e->getMessage()}");
}
break;
case SERVER_SNAPSHOT_CREATION_STATUS::CREATING_ROLE:
try {
if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_ALL) {
$saveOldRole = false;
try {
$dbRole = DBRole::loadById($DBServer->roleId);
if ($dbRole->name == $BundleTask->roleName && $dbRole->envId == $BundleTask->envId) {
$saveOldRole = true;
}
} catch (Exception $e) {
//NO OLD ROLE
}
if ($dbRole && $saveOldRole) {
if ($DBServer) {
$new_role_name = BundleTask::GenerateRoleName($DBServer->GetFarmRoleObject(), $DBServer);
} else {
$new_role_name = $BundleTask->roleName . "-" . rand(1000, 9999);
}
$dbRole->name = $new_role_name;
$BundleTask->Log(sprintf(_("Old role '%s' (ID: %s) renamed to '%s'"), $BundleTask->roleName, $dbRole->id, $new_role_name));
$dbRole->save();
} else {
//TODO:
//$this->Logger->error("dbRole->replace->fail({$BundleTask->roleName}, {$BundleTask->envId})");
}
}
try {
$DBRole = DBRole::createFromBundleTask($BundleTask);
} catch (Exception $e) {
$BundleTask->SnapshotCreationFailed("Role creation failed due to internal error ({$e->getMessage()}). Please try again.");
return;
}
if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::NO_REPLACE) {
$BundleTask->setDate('finished');
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS;
$BundleTask->Log(sprintf(_("Replacement type: %s. Bundle task status: %s"), SERVER_REPLACEMENT_TYPE::NO_REPLACE, SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS));
try {
$DBServer = DBServer::LoadByID($BundleTask->serverId);
if ($DBServer->status == SERVER_STATUS::IMPORTING) {
if ($DBServer->farmId) {
// Create DBFarmRole object
// TODO: create DBFarm role
}
//$DBServer->Delete();
}
} catch (Exception $e) {
}
} else {
try {
$BundleTask->Log(sprintf(_("Replacement type: %s. Bundle task status: %s"), $BundleTask->replaceType, SERVER_SNAPSHOT_CREATION_STATUS::REPLACING_SERVERS));
if ($BundleTask->replaceType == SERVER_REPLACEMENT_TYPE::REPLACE_FARM) {
$DBFarm = DBFarm::LoadByID($BundleTask->farmId);
$DBFarmRole = $DBFarm->GetFarmRoleByRoleID($BundleTask->prototypeRoleId);
$DBFarmRole->NewRoleID = $BundleTask->roleId;
$DBFarmRole->Save();
} else {
$farm_roles = $db->GetAll("SELECT id FROM farm_roles WHERE role_id=? AND farmid IN (SELECT id FROM farms WHERE env_id=?)", array($BundleTask->prototypeRoleId, $BundleTask->envId));
foreach ($farm_roles as $farm_role) {
$DBFarmRole = DBFarmRole::LoadByID($farm_role['id']);
$DBFarmRole->NewRoleID = $BundleTask->roleId;
$DBFarmRole->Save();
}
}
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::REPLACING_SERVERS;
} catch (Exception $e) {
$this->Logger->error($e->getMessage());
$BundleTask->Log(sprintf(_("Server replacement failed: %s"), $e->getMessage()));
$BundleTask->setDate('finished');
$BundleTask->status = SERVER_SNAPSHOT_CREATION_STATUS::SUCCESS;
}
}
$BundleTask->Save();
} catch (Exception $e) {
$this->Logger->error($e->getMessage());
}
break;
}
}
public function limitsAction()
{
$locations = $p = PlatformFactory::NewPlatform(SERVER_PLATFORMS::RACKSPACE)->getLocations();
$this->response->page('ui/tools/rackspace/limits.js', array('locations' => self::loadController('Platforms')->getCloudLocations(SERVER_PLATFORMS::RACKSPACE, false)));
}
private function getCloudLocationName($platform, $cloudLocation)
{
$cloudLocationName = $cloudLocation;
$locations = PlatformFactory::NewPlatform($platform)->getLocations();
if (isset($locations[$cloudLocation])) {
$cloudLocationName = $locations[$cloudLocation];
}
return $cloudLocationName;
}
if ($DBServer->farmId != $DBFarm->ID) {
throw new Exception("Server not found");
}
if ($DBServer->GetProperty(SERVER_PROPERTIES::SZR_VESION) != $pkg_ver) {
$DBServer->SetProperty(SERVER_PROPERTIES::SZR_VESION, $pkg_ver);
}
/**
* Deserialize data from instance
*/
if (!$DBServer->localIp) {
$DBServer->localIp = $data["localip"];
}
if ($DBServer->localIp == $_SERVER['REMOTE_ADDR'] && $DBServer->platform == SERVER_PLATFORMS::EC2) {
if (!$DBServer->remoteIp) {
try {
$ips = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
$_SERVER['REMOTE_ADDR'] = $ips['remoteIp'];
$Logger->info(sprintf("Instance external ip = '%s'", $_SERVER['REMOTE_ADDR']));
} catch (Exception $e) {
$Logger->fatal(sprintf(_("Cannot determine external IP for instance %s: %s"), $req_InstanceID, $e->getMessage()));
exit;
}
} else {
$_SERVER['REMOTE_ADDR'] = $DBServer->remoteIp;
}
}
//************************//
if ($DBFarm && $DBServer) {
$instance_events = array("hostInit" => "HostInit", "hostUp" => "HostUp", "rebootFinish" => "RebootComplete", "IPAddressChanged" => "IPAddressChanged", "newMysqlMaster" => "NewMysqlMasterUp");
switch ($req_EventType) {
case "go2Halt":
public function SnapshotCreationFailed($failed_reason)
{
if ($this->status == SERVER_SNAPSHOT_CREATION_STATUS::REPLACING_SERVERS) {
$replacingServers = true;
}
$this->status = SERVER_SNAPSHOT_CREATION_STATUS::FAILED;
$this->failureReason = $failed_reason;
if ($this->farmId) {
try {
$DBFarm = DBFarm::LoadByID($this->farmId);
if ($DBFarm->Status == FARM_STATUS::SYNCHRONIZING && !$DBFarm->TermOnSyncFail) {
$this->Db->Execute("UPDATE farms SET status=? WHERE id=?", array(FARM_STATUS::RUNNING, $this->farmId));
$this->Log(sprintf(_("Farm status set to Running")));
}
} catch (Exception $e) {
}
}
try {
$dbServer = DBServer::LoadByID($this->serverId);
if ($replacingServers) {
try {
$dbFarmRole = $dbServer->GetFarmRoleObject();
$dbFarmRole->NewRoleID = NULL;
$dbFarmRole->Save();
$dbServer->roleId = $dbFarmRole->RoleID;
$dbServer->Save();
$this->Db->Execute("UPDATE servers SET status=? WHERE role_id = ? AND replace_server_id IS NOT NULL AND farm_roleid=?", array(SERVER_STATUS::PENDING_TERMINATE, $this->roleId, $dbFarmRole->ID));
} catch (Exception $e) {
//TODO:
}
}
if ($dbServer->status == SERVER_STATUS::TEMPORARY) {
try {
if ($dbServer->GetProperty(EC2_SERVER_PROPERTIES::INSTANCE_ID)) {
PlatformFactory::NewPlatform($dbServer->platform)->TerminateServer($dbServer);
}
} catch (Exception $e) {
}
}
} catch (Exception $e) {
}
$this->Log(sprintf(_("Snapshot creation failed. Reason: %s. Bundle task status changed to: %s"), $failed_reason, $this->status));
$this->Save();
}
function handleWork($farmId)
{
$this->cleanup();
$DBFarm = DBFarm::LoadByID($farmId);
$account = Scalr_Account::init()->loadById($DBFarm->ClientID);
$payAsYouGoTime = $account->getSetting(Scalr_Account::SETTING_BILLING_PAY_AS_YOU_GO_DATE);
$GLOBALS["SUB_TRANSACTIONID"] = abs(crc32(posix_getpid() . $farmId));
$GLOBALS["LOGGER_FARMID"] = $farmId;
$this->logger->info("[" . $GLOBALS["SUB_TRANSACTIONID"] . "] Begin polling farm (ID: {$DBFarm->ID}, Name: {$DBFarm->Name}, Status: {$DBFarm->Status})");
//
// Collect information from database
//
$servers_count = $this->db->GetOne("SELECT COUNT(*) FROM servers WHERE farm_id = ? AND status != ?", array($DBFarm->ID, SERVER_STATUS::TERMINATED));
$this->logger->info("[FarmID: {$DBFarm->ID}] Found {$servers_count} farm instances in database");
if ($DBFarm->Status == FARM_STATUS::TERMINATED && $servers_count == 0) {
return;
}
foreach ($DBFarm->GetServersByFilter(array(), array('status' => SERVER_STATUS::PENDING_LAUNCH)) as $DBServer) {
try {
if ($DBServer->status != SERVER_STATUS::PENDING && $DBServer->status != SERVER_STATUS::PENDING_TERMINATE) {
$p = PlatformFactory::NewPlatform($DBServer->platform);
if (!$p->IsServerExists($DBServer)) {
try {
$serverInfo = $p->GetServerExtendedInformation($DBServer);
} catch (Exception $e) {
Logger::getLogger(LOG_CATEGORY::FARM)->error(sprintf("[CRASH][FarmID: {$DBFarm->ID}] Crash check for server '{$DBServer->serverId}' failed: {$e->getMessage()}"));
}
if (!$serverInfo) {
if ($DBServer->status != SERVER_STATUS::PENDING_TERMINATE && $DBServer->status != SERVER_STATUS::TERMINATED) {
if ($DBServer->GetProperty("system.crashed") == 1) {
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
Scalr::FireEvent($DBFarm->ID, new HostCrashEvent($DBServer));
} else {
$DBServer->SetProperty(SERVER_PROPERTIES::REBOOTING, 0);
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' found in database but not found on {$DBServer->platform}. Crashed.", $DBServer->serverId)));
$DBServer->SetProperty("system.crashed", "1");
}
continue;
}
} else {
Logger::getLogger(LOG_CATEGORY::FARM)->error(sprintf("[CRASH][FarmID: {$DBFarm->ID}] False-positive crash check: {$DBServer->serverId}"));
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
Logger::getLogger(LOG_CATEGORY::FARM)->fatal(sprintf("[CRASH][FarmID: {$DBFarm->ID}] InstanceID: %s, List: %s", $DBServer->GetCloudServerID(), json_encode($p->instancesListCache)));
}
}
} else {
$DBServer->SetProperty("system.crashed", "0");
}
}
} catch (Exception $e) {
if (stristr($e->getMessage(), "AWS was not able to validate the provided access credentials") || stristr($e->getMessage(), "Unable to sign AWS API request. Please, check your X.509")) {
$env = Scalr_Environment::init()->LoadById($DBFarm->EnvID);
$env->status = Scalr_Environment::STATUS_INACTIVE;
$env->save();
$env->setPlatformConfig(array('system.auto-disable-reason' => $e->getMessage()), false);
return;
}
if (stristr($e->getMessage(), "Could not connect to host")) {
continue;
}
print "[0][Farm: {$farmId}] {$e->getMessage()} at {$e->getFile()}:{$e->getLine()}\n\n";
continue;
}
try {
$realStatus = $DBServer->GetRealStatus()->getName();
if ($realStatus == 'stopped') {
$DBServer->SetProperty(SERVER_PROPERTIES::SUB_STATUS, $realStatus);
continue;
} else {
if ($DBServer->GetProperty(SERVER_PROPERTIES::SUB_STATUS) == 'stopped') {
$DBServer->SetProperty(SERVER_PROPERTIES::SUB_STATUS, "");
}
}
} catch (Exception $e) {
}
try {
if (!in_array($DBServer->status, array(SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE)) && $DBServer->GetRealStatus()->isTerminated()) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) not running (Real state: %s, Scalr status: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName(), $DBServer->status)));
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
$DBServer->SetProperty(SERVER_PROPERTIES::REBOOTING, 0);
Scalr::FireEvent($DBFarm->ID, new HostDownEvent($DBServer));
continue;
} elseif ($DBServer->status != SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->IsRunning()) {
if ($DBServer->status != SERVER_STATUS::TERMINATED && $DBServer->status != SERVER_STATUS::TROUBLESHOOTING) {
/*
if ($DBServer->platform == SERVER_PLATFORMS::NIMBULA)
{
if (!$DBServer->GetProperty(NIMBULA_SERVER_PROPERTIES::USER_DATA_INJECTED))
{
$dbRole = $DBServer->GetFarmRoleObject()->GetRoleObject();
$ssh2Client = new Scalr_Net_Ssh2_Client();
$ssh2Client->addPassword(
$dbRole->getProperty(DBRole::PROPERTY_NIMBULA_INIT_ROOT_USER),
$dbRole->getProperty(DBRole::PROPERTY_NIMBULA_INIT_ROOT_PASS)
);
$info = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
$port = $dbRole->getProperty(DBRole::PROPERTY_SSH_PORT);
if (!$port) $port = 22;
try {
$ssh2Client->connect($info['remoteIp'], $port);
foreach ($DBServer->GetCloudUserData() as $k=>$v)
$u_data .= "{$k}={$v};";
$u_data = trim($u_data, ";");
$ssh2Client->sendFile('/etc/scalr/private.d/.user-data', $u_data, "w+", false);
$DBServer->SetProperty(NIMBULA_SERVER_PROPERTIES::USER_DATA_INJECTED, 1);
}
catch(Exception $e) {
Logger::getLogger(LOG_CATEGORY::FARM)->error(new FarmLogMessage($DBFarm->ID, $e->getMessage()));
}
}
}
*/
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
if ($DBServer->status == SERVER_STATUS::PENDING && $DBFarm->GetSetting(DBFarm::SETTING_EC2_VPC_ID)) {
if ($DBServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_AWS_VPC_INTERNET_ACCESS) != 'outbound-only') {
$ipAddress = Modules_Platforms_Ec2_Helpers_Eip::setEipForServer($DBServer);
if ($ipAddress) {
$DBServer->remoteIp = $ipAddress;
$DBServer->Save();
}
}
}
}
if ($DBServer->isOpenstack()) {
$ipPool = $DBServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_OPENSTACK_IP_POOL);
if (!$DBServer->remoteIp && ($DBServer->status == SERVER_STATUS::PENDING || $DBServer->status == SERVER_STATUS::INIT) && $ipPool) {
$osClient = $DBServer->GetEnvironmentObject()->openstack($DBServer->platform, $DBServer->GetProperty(OPENSTACK_SERVER_PROPERTIES::CLOUD_LOCATION));
if ($osClient->hasService('network')) {
$platform = PlatformFactory::NewPlatform($DBServer->platform);
$serverIps = $platform->GetServerIPAddresses($DBServer);
/********* USE Quantum (Neuron) NETWORK *******/
$ips = $osClient->network->floatingIps->list();
//Check free existing IP
$ipAssigned = false;
$ipAddress = false;
$ipId = false;
$ipInfo = false;
foreach ($ips as $ip) {
if ($ip->fixed_ip_address == $serverIps['localIp'] && $ip->port_id) {
$ipAddress = $ip->floating_ip_address;
$ipId = $ip->id;
$ipAssigned = true;
$ipInfo = $ip;
break;
}
/*
if (!$ip->fixed_ip_address && !$ipAddress) {
$ipAddress = $ip->floating_ip_address;
$ipId = $ip->id;
$ipInfo = $ip;
}
*/
}
if ($ipInfo) {
Logger::getLogger("Openstack")->warn(new FarmLogMessage($DBServer->farmId, "Found free floating IP: {$ipAddress} for use (" . json_encode($ipInfo) . ")"));
}
if (!$ipAssigned) {
// Get instance port
$ports = $osClient->network->ports->list();
foreach ($ports as $port) {
if ($port->device_id == $DBServer->GetProperty(OPENSTACK_SERVER_PROPERTIES::SERVER_ID)) {
$serverNetworkPort = $port->id;
break;
}
}
if (!$serverNetworkPort) {
Logger::getLogger("Openstack")->error(new FarmLogMessage($DBServer->farmId, "Unable to identify network port of instance"));
} else {
if (!$ipAddress) {
$publicNetworkId = $ipPool;
/*
$networks = $osClient->network->listNetworks();
foreach ($networks as $network) {
if ($network->{"router:external"} == true) {
$publicNetworkId = $network->id;
}
}
if (!$publicNetworkId) {
Logger::getLogger("Openstack")->error(new FarmLogMessage($DBServer->farmId,
"Unable to identify public network to allocate"
));
} else {
*/
$ip = $osClient->network->floatingIps->create($publicNetworkId, $serverNetworkPort);
$ipAddress = $ip->floating_ip_address;
$DBServer->SetProperties(array(OPENSTACK_SERVER_PROPERTIES::FLOATING_IP => $ip->floating_ip_address, OPENSTACK_SERVER_PROPERTIES::FLOATING_IP_ID => $ip->id));
Logger::getLogger("Openstack")->warn(new FarmLogMessage($DBServer->farmId, "Allocated new IP {$ipAddress} for port: {$serverNetworkPort}"));
//}
} else {
/*
Logger::getLogger("Openstack")->warn(new FarmLogMessage($DBServer->farmId,
"Updating IP {$ipAddress} ({$ipId}) with port: {$serverNetworkPort}"
));
$osClient->network->floatingIps->update($ipId, $serverNetworkPort);
*/
}
}
} else {
Logger::getLogger("Openstack")->warn(new FarmLogMessage($DBServer->farmId, "IP: {$ipAddress} already assigned"));
}
} else {
/********* USE NOVA NETWORK *******/
//Check free existing IP
$ipAssigned = false;
$ipAddress = false;
$ips = $osClient->servers->floatingIps->list($ipPool);
foreach ($ips as $ip) {
//if (!$ip->instance_id) {
// $ipAddress = $ip->ip;
//}
if ($ip->instance_id == $DBServer->GetProperty(OPENSTACK_SERVER_PROPERTIES::SERVER_ID)) {
$ipAddress = $ip->ip;
$ipAssigned = true;
}
}
//If no free IP allocate new from pool
if (!$ipAddress) {
$ip = $osClient->servers->floatingIps->create($ipPool);
$ipAddress = $ip->ip;
$DBServer->SetProperties(array(OPENSTACK_SERVER_PROPERTIES::FLOATING_IP => $ip->ip, OPENSTACK_SERVER_PROPERTIES::FLOATING_IP_ID => $ip->id));
}
if (!$ipAssigned) {
//Associate floating IP with Instance
$osClient->servers->addFloatingIp($DBServer->GetCloudServerID(), $ipAddress);
}
}
if ($ipAddress) {
$DBServer->remoteIp = $ipAddress;
$DBServer->Save();
$DBServer->SetProperty(SERVER_PROPERTIES::SYSTEM_IGNORE_INBOUND_MESSAGES, null);
}
}
}
if ($DBServer->isCloudstack()) {
if ($DBServer->status == SERVER_STATUS::PENDING) {
$jobId = $DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::LAUNCH_JOB_ID);
try {
$platform = PlatformFactory::NewPlatform($DBServer->platform);
$cs = Scalr_Service_Cloud_Cloudstack::newCloudstack($platform->getConfigVariable(Modules_Platforms_Cloudstack::API_URL, $DBServer->GetEnvironmentObject()), $platform->getConfigVariable(Modules_Platforms_Cloudstack::API_KEY, $DBServer->GetEnvironmentObject()), $platform->getConfigVariable(Modules_Platforms_Cloudstack::SECRET_KEY, $DBServer->GetEnvironmentObject()), $DBServer->platform);
$res = $cs->queryAsyncJobResult($jobId);
if ($res->jobresult->jobstatus == 1) {
$DBServer->SetProperty(CLOUDSTACK_SERVER_PROPERTIES::TMP_PASSWORD, $res->jobresult->virtualmachine->password);
$DBServer->SetProperty(CLOUDSTACK_SERVER_PROPERTIES::SERVER_NAME, $res->jobresult->virtualmachine->name);
}
//TODO: handle failed job: $res->jobresult->jobstatus == 2
} catch (Exception $e) {
if ($DBServer->farmId) {
Logger::getLogger("CloudStack")->error(new FarmLogMessage($DBServer->farmId, $e->getMessage()));
}
}
}
}
try {
$dtadded = strtotime($DBServer->dateAdded);
$DBFarmRole = $DBServer->GetFarmRoleObject();
$launch_timeout = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) > 0 ? $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) : 900;
} catch (Exception $e) {
if (stristr($e->getMessage(), "not found")) {
$DBServer->terminate('FARM_ROLE_NOT_FOUND');
}
}
$scripting_event = false;
if ($DBServer->status == SERVER_STATUS::PENDING) {
$event = "hostInit";
$scripting_event = EVENT_TYPE::HOST_INIT;
} elseif ($DBServer->status == SERVER_STATUS::INIT) {
$event = "hostUp";
$scripting_event = EVENT_TYPE::HOST_UP;
}
if ($scripting_event && $dtadded) {
$scripting_timeout = (int) $this->db->GetOne("\n SELECT sum(timeout)\n FROM farm_role_scripts\n WHERE event_name=? AND\n farm_roleid=? AND issync='1'\n ", array($scripting_event, $DBServer->farmRoleId));
if ($scripting_timeout) {
$launch_timeout = $launch_timeout + $scripting_timeout;
}
if ($dtadded + $launch_timeout < time() && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
//Add entry to farm log
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' did not send '%s' event in %s seconds after launch (Try increasing timeouts in role settings). Considering it broken. Terminating instance.", $DBServer->serverId, $event, $launch_timeout)));
try {
$DBServer->terminate(array('SERVER_DID_NOT_SEND_EVENT', $event, $launch_timeout), false);
} catch (Exception $err) {
$this->logger->fatal($err->getMessage());
}
} elseif ($DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
//DO NOT TERMINATE MONGODB INSTANCES BY TIMEOUT! IT'S NOT SAFE
//THINK ABOUT WORKAROUND
}
}
// Is IP address changed?
if (!$DBServer->IsRebooting()) {
$ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
if ($ipaddresses['remoteIp'] && $DBServer->remoteIp && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp && $DBServer->localIp != $ipaddresses['localIp']) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("RemoteIP: %s (%s), LocalIp: %s (%s) (Poller).", $DBServer->remoteIp, $ipaddresses['remoteIp'], $DBServer->localIp, $ipaddresses['localIp'])));
Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp']));
}
//TODO: Check health:
}
}
} elseif ($DBServer->status == SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->isRunning()) {
// Is IP address changed?
if (!$DBServer->IsRebooting()) {
$ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
if ($ipaddresses['remoteIp'] && $DBServer->remoteIp && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp && $DBServer->localIp != $ipaddresses['localIp']) {
Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp']));
}
if ($payAsYouGoTime) {
$initTime = $DBServer->GetProperty(SERVER_PROPERTIES::INITIALIZED_TIME);
if ($initTime < $payAsYouGoTime) {
$initTime = $payAsYouGoTime;
}
$runningHours = ceil((time() - $initTime) / 3600);
$scuUsed = $runningHours * Scalr_Billing::getSCUByInstanceType($DBServer->GetFlavor());
$this->db->Execute("UPDATE servers_history SET scu_used = ?, scu_updated = 0 WHERE server_id = ?", array($scuUsed, $DBServer->serverId));
}
//Update GCE ServerID
if ($DBServer->platform == SERVER_PLATFORMS::GCE) {
if ($DBServer->GetProperty(GCE_SERVER_PROPERTIES::SERVER_ID) == $DBServer->serverId) {
$info = PlatformFactory::NewPlatform($DBServer->platform)->GetServerExtendedInformation($DBServer);
$DBServer->SetProperty(GCE_SERVER_PROPERTIES::SERVER_ID, $info['Cloud Server ID']);
}
}
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
$env = Scalr_Environment::init()->loadById($DBServer->envId);
$ec2 = $env->aws($DBServer->GetCloudLocation())->ec2;
//TODO:
$isEnabled = $ec2->instance->describeAttribute($DBServer->GetCloudServerID(), InstanceAttributeType::disableApiTermination());
$DBServer->SetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED, $isEnabled);
}
} else {
//TODO: Check reboot timeout
}
}
} catch (Exception $e) {
if (stristr($e->getMessage(), "not found")) {
print $e->getTraceAsString() . "\n";
} else {
print "[1][Farm: {$farmId}] {$e->getMessage()} at {$e->getFile()}:{$e->getLine()}\n\n";
}
}
}
}
public function remove($removeImage = false)
{
if ($removeImage) {
$platforms = array_keys($this->getImages());
foreach ($platforms as $platform) {
PlatformFactory::NewPlatform($platform)->RemoveServerSnapshot($this);
}
}
$this->db->Execute("DELETE FROM roles WHERE id = ?", array($this->id));
$this->db->Execute("DELETE FROM roles_queue WHERE role_id = ?", array($this->id));
}
public function RemoveServerSnapshot(DBRole $DBRole)
{
foreach (PlatformFactory::getOpenstackBasedPlatforms() as $platform) {
foreach ($DBRole->getImageId($platform) as $location => $imageId) {
try {
$osClient = $DBRole->getEnvironmentObject()->openstack($platform, $location);
$osClient->servers->images->delete($imageId);
} catch (Exception $e) {
if (stristr($e->getMessage(), "Unavailable service \"compute\" or region") || stristr($e->getMessage(), "Image not found") || stristr($e->getMessage(), "Cannot destroy a destroyed snapshot") || stristr($e->getMessage(), "OpenStack error. Could not find user")) {
//DO NOTHING
} else {
throw $e;
}
}
}
}
return true;
}
public function edit2Action()
{
$this->request->restrictAccess(Acl::RESOURCE_FARMS_ROLES, Acl::PERM_FARMS_ROLES_MANAGE);
$this->request->defineParams(array('roleId' => array('type' => 'int')));
$params = array('platforms' => array());
$platforms = $this->user->isScalrAdmin() ? (array) \Scalr::config('scalr.allowed_clouds') : $this->getEnvironment()->getEnabledPlatforms();
foreach ($platforms as $platform) {
$params['platforms'][$platform] = array('locations' => array());
if ($platform !== 'gce') {
foreach (PlatformFactory::NewPlatform($platform)->getLocations() as $lk => $lv) {
$params['platforms'][$platform]['locations'][$lk] = $lv;
}
}
}
$params['scriptData'] = self::loadController('Scripts')->getScriptingData();
$params['categories'] = $this->db->GetAll("SELECT * FROM role_categories WHERE env_id='0'", array());
if ($this->getParam('roleId')) {
$dbRole = DBRole::loadById($this->getParam('roleId'));
if (!$this->user->isScalrAdmin()) {
$this->user->getPermissions()->validate($dbRole);
}
$images = array();
$imDetails = $dbRole->getImages(true);
if (!empty($imDetails) && (is_array($imDetails) || $imDetails instanceof \Traversable)) {
foreach ($imDetails as $platform => $locations) {
foreach ($locations as $location => $imageInfo) {
$images[] = array('image_id' => $imageInfo['image_id'], 'platform' => $platform, 'location' => $location, 'architecture' => $imageInfo['architecture']);
}
}
}
$params['role'] = array('roleId' => $dbRole->id, 'name' => $dbRole->name, 'catId' => $dbRole->catId, 'os' => $dbRole->os, 'osFamily' => $dbRole->osFamily, 'osGeneration' => $dbRole->osGeneration, 'osVersion' => $dbRole->osVersion, 'description' => $dbRole->description, 'behaviors' => $dbRole->getBehaviors(), 'images' => $images, 'scripts' => $dbRole->getScripts(), 'dtadded' => Scalr_Util_DateTime::convertTz($dbRole->{$dbRole}->dateAdded), 'addedByEmail' => $dbRole->addedByEmail, 'software' => $dbRole->getSoftwareList(), 'tags' => array_flip($dbRole->getTags()));
if ($dbRole->origin == ROLE_TYPE::CUSTOM) {
$variables = new Scalr_Scripting_GlobalVariables($this->getEnvironmentId(), Scalr_Scripting_GlobalVariables::SCOPE_ROLE);
$params['role']['variables'] = json_encode($variables->getValues($dbRole->id));
}
} else {
$params['role'] = array('roleId' => 0, 'name' => '', 'arch' => 'x86_64', 'agent' => 2, 'description' => '', 'behaviors' => array(), 'images' => array(), 'scripts' => array(), 'tags' => array());
}
$this->response->page('ui/roles/edit2.js', $params, array('ui/roles/edit/overview.js', 'ui/roles/edit/images.js', 'ui/roles/edit/scripting.js', 'ui/roles/edit/variables.js', 'ui/roles/edit/chef.js', 'ui/scripts/scriptfield2.js', 'ui/core/variablefield.js', 'ux-boxselect.js', 'ui/services/chef/chefsettings.js'), array('ui/roles/edit2.css', 'ui/scripts/scriptfield2.css', 'ui/core/variablefield.css'));
}
/**
* Launches server
*
* @param \ServerCreateInfo $ServerCreateInfo optional The server create info
* @param \DBServer $DBServer optional The DBServer object
* @param bool $delayed optional
* @param string $reason optional
* @param \Scalr_Account_User|int $user optional The Scalr_Account_User object or its unique identifier
* @return DBServer|null Returns the DBServer object on cussess or null otherwise
*/
public static function LaunchServer(ServerCreateInfo $ServerCreateInfo = null, DBServer $DBServer = null, $delayed = false, $reason = "", $user = null)
{
$db = self::getDb();
//Ensures handling identifier of the user instead of the object
if ($user !== null && !$user instanceof \Scalr_Account_User) {
try {
$user = Scalr_Account_User::init()->loadById(intval($user));
} catch (\Exception $e) {
}
}
if (!$DBServer && $ServerCreateInfo) {
$ServerCreateInfo->SetProperties(array(SERVER_PROPERTIES::SZR_KEY => Scalr::GenerateRandomKey(40), SERVER_PROPERTIES::SZR_KEY_TYPE => SZR_KEY_TYPE::ONE_TIME));
$DBServer = DBServer::Create($ServerCreateInfo, false, true);
} elseif (!$DBServer && !$ServerCreateInfo) {
// incorrect arguments
Logger::getLogger(LOG_CATEGORY::FARM)->error(sprintf("Cannot create server"));
return null;
}
if ($user instanceof \Scalr_Account_User) {
$DBServer->SetProperties(array(SERVER_PROPERTIES::LAUNCHED_BY_ID => $user->id, SERVER_PROPERTIES::LAUNCHED_BY_EMAIL => $user->getEmail()));
}
if ($delayed) {
$DBServer->status = SERVER_STATUS::PENDING_LAUNCH;
$DBServer->SetProperty(SERVER_PROPERTIES::LAUNCH_REASON, $reason);
$DBServer->Save();
return $DBServer;
}
if ($ServerCreateInfo && $ServerCreateInfo->roleId) {
$dbRole = DBRole::loadById($ServerCreateInfo->roleId);
if ($dbRole->generation == 1) {
$DBServer->status = SERVER_STATUS::PENDING_LAUNCH;
$DBServer->Save();
$DBServer->SetProperty(SERVER_PROPERTIES::LAUNCH_ERROR, "ami-scripts servers no longer supported");
return $DBServer;
}
}
try {
$account = Scalr_Account::init()->loadById($DBServer->clientId);
$account->validateLimit(Scalr_Limits::ACCOUNT_SERVERS, 1);
PlatformFactory::NewPlatform($DBServer->platform)->LaunchServer($DBServer);
$DBServer->status = SERVER_STATUS::PENDING;
$DBServer->Save();
try {
if (!$reason) {
$reason = $DBServer->GetProperty(SERVER_PROPERTIES::LAUNCH_REASON);
}
$DBServer->getServerHistory()->markAsLaunched($reason);
} catch (Exception $e) {
Logger::getLogger('SERVER_HISTORY')->error(sprintf("Cannot update servers history: {$e->getMessage()}"));
}
} catch (Exception $e) {
Logger::getLogger(LOG_CATEGORY::FARM)->error(new FarmLogMessage($DBServer->farmId, sprintf("Cannot launch server on '%s' platform: %s", $DBServer->platform, $e->getMessage())));
$DBServer->status = SERVER_STATUS::PENDING_LAUNCH;
$DBServer->SetProperty(SERVER_PROPERTIES::LAUNCH_ERROR, $e->getMessage());
$DBServer->Save();
}
if ($DBServer->status == SERVER_STATUS::PENDING) {
Scalr::FireEvent($DBServer->farmId, new BeforeInstanceLaunchEvent($DBServer));
$DBServer->SetProperty(SERVER_PROPERTIES::LAUNCH_ERROR, "");
}
return $DBServer;
}
function handleWork($farmId)
{
$this->cleanup();
$DBFarm = DBFarm::LoadByID($farmId);
$GLOBALS["SUB_TRANSACTIONID"] = abs(crc32(posix_getpid() . $farmId));
$GLOBALS["LOGGER_FARMID"] = $farmId;
$this->logger->info("[" . $GLOBALS["SUB_TRANSACTIONID"] . "] Begin polling farm (ID: {$DBFarm->ID}, Name: {$DBFarm->Name}, Status: {$DBFarm->Status})");
//
// Collect information from database
//
$servers_count = $this->db->GetOne("SELECT COUNT(*) FROM servers WHERE farm_id = ? AND status != ?", array($DBFarm->ID, SERVER_STATUS::TERMINATED));
$this->logger->info("[FarmID: {$DBFarm->ID}] Found {$servers_count} farm instances in database");
if ($DBFarm->Status == FARM_STATUS::TERMINATED && $servers_count == 0) {
return;
}
/*
O:8:"stdClass":1:{s:7:"servers";a:3:{i:0;O:8:"stdClass":9:{s:8:"progress";i:100;s:2:"id";i:20842485;s:7:"imageId";i:77;s:8:"flavorId";i:3;s:6:"status";s:6:"ACTIVE";s:4:"name";s:36:"4d63c9c4-6f2d-4c16-9c1e-623ef2b70da8";s:6:"hostId";s:32:"1de22da589d35371f10ef232b77a6ede";s:9:"addresses";O:8:"stdClass":2:{s:6:"public";a:1:{i:0;s:13:"50.57.122.122";}s:7:"private";a:1:{i:0;s:13:"10.182.160.86";}}s:8:"metadata";O:8:"stdClass":0:{}}i:1;O:8:"stdClass":9:{s:8:"progress";i:100;s:2:"id";i:20842486;s:7:"imageId";i:77;s:8:"flavorId";i:3;s:6:"status";s:6:"ACTIVE";s:4:"name";s:36:"df6f3f67-0cfb-482d-9307-3d394dd009a1";s:6:"hostId";s:32:"00a70eb9f5b7e667d4f1ff3d8c5ef496";s:9:"addresses";O:8:"stdClass":2:{s:6:"public";a:1:{i:0;s:13:"50.57.122.131";}s:7:"private";a:1:{i:0;s:13:"10.182.160.89";}}s:8:"metadata";O:8:"stdClass":0:{}}i:2;O:8:"stdClass":9:{s:8:"progress";i:100;s:2:"id";i:20842516;s:7:"imageId";i:77;s:8:"flavorId";i:3;s:6:"status";s:6:"ACTIVE";s:4:"name";s:36:"27128bc2-1da8-4cef-b527-0b916309ae7e";s:6:"hostId";s:32:"254b44d9672258f1d7d189ab85627842";s:9:"addresses";O:8:"stdClass":2:{s:6:"public";a:1:{i:0;s:13:"50.57.122.140";}s:7:"private";a:1:{i:0;s:14:"10.182.160.112";}}s:8:"metadata";O:8:"stdClass":0:{}}}}
O:8:"stdClass":1:{s:7:"servers";a:3:{i:0;O:8:"stdClass":9:{s:8:"progress";i:100;s:2:"id";i:20842485;s:7:"imageId";i:77;s:8:"flavorId";i:3;s:6:"status";s:6:"ACTIVE";s:4:"name";s:36:"4d63c9c4-6f2d-4c16-9c1e-623ef2b70da8";s:6:"hostId";s:32:"1de22da589d35371f10ef232b77a6ede";s:9:"addresses";O:8:"stdClass":2:{s:6:"public";a:1:{i:0;s:13:"50.57.122.122";}s:7:"private";a:1:{i:0;s:13:"10.182.160.86";}}s:8:"metadata";O:8:"stdClass":0:{}}i:1;O:8:"stdClass":9:{s:8:"progress";i:100;s:2:"id";i:20842486;s:7:"imageId";i:77;s:8:"flavorId";i:3;s:6:"status";s:6:"ACTIVE";s:4:"name";s:36:"df6f3f67-0cfb-482d-9307-3d394dd009a1";s:6:"hostId";s:32:"00a70eb9f5b7e667d4f1ff3d8c5ef496";s:9:"addresses";O:8:"stdClass":2:{s:6:"public";a:1:{i:0;s:13:"50.57.122.131";}s:7:"private";a:1:{i:0;s:13:"10.182.160.89";}}s:8:"metadata";O:8:"stdClass":0:{}}i:2;O:8:"stdClass":9:{s:8:"progress";i:100;s:2:"id";i:20842516;s:7:"imageId";i:77;s:8:"flavorId";i:3;s:6:"status";s:6:"ACTIVE";s:4:"name";s:36:"27128bc2-1da8-4cef-b527-0b916309ae7e";s:6:"hostId";s:32:"254b44d9672258f1d7d189ab85627842";s:9:"addresses";O:8:"stdClass":2:{s:6:"public";a:1:{i:0;s:13:"50.57.122.140";}s:7:"private";a:1:{i:0;s:14:"10.182.160.112";}}s:8:"metadata";O:8:"stdClass":0:{}}}}
*/
foreach ($DBFarm->GetServersByFilter(array(), array('status' => SERVER_STATUS::PENDING_LAUNCH)) as $DBServer) {
try {
if ($DBServer->status != SERVER_STATUS::PENDING) {
$p = PlatformFactory::NewPlatform($DBServer->platform);
if (!$p->IsServerExists($DBServer)) {
if ($DBServer->platform == SERVER_PLATFORMS::RACKSPACE) {
if ($DBServer->status != SERVER_STATUS::PENDING_TERMINATE && $DBServer->status != SERVER_STATUS::TERMINATED) {
//TODO:
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' found in database but not found on {$DBServer->platform}. Crashed. RACKSPACE, doing nothing.", $DBServer->serverId)));
mail("*****@*****.**", "RACKSPACE - {$DBServer->serverId} ({$DBServer->remoteIp}) ({$DBServer->GetCloudServerID()}) ({$DBServer->GetProperty(RACKSPACE_SERVER_PROPERTIES::HOST_ID)})", serialize($p->_tmpVar));
$DBServer->SetProperty("rackspace.crashed", "1");
continue;
}
} else {
if ($DBServer->status != SERVER_STATUS::TERMINATED && $DBServer->status != SERVER_STATUS::PENDING_TERMINATE) {
$DBServer->SetProperty(SERVER_PROPERTIES::REBOOTING, 0);
// Add entry to farm log
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' found in database but not found on {$DBServer->platform}. Crashed.", $DBServer->serverId)));
Scalr::FireEvent($DBFarm->ID, new HostCrashEvent($DBServer));
continue;
}
}
}
}
} catch (Exception $e) {
if (stristr($e->getMessage(), "AWS was not able to validate the provided access credentials")) {
continue;
}
if (stristr($e->getMessage(), "Could not connect to host")) {
continue;
}
print "[Farm: {$farmId}] {$e->getMessage()} at {$e->getFile()}:{$e->getLine()}\n\n";
continue;
}
try {
if ($DBServer->status != SERVER_STATUS::TERMINATED && $DBServer->GetRealStatus()->isTerminated()) {
if ($DBServer->status != SERVER_STATUS::PENDING_TERMINATE) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) not running (Real state: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName())));
}
$DBServer->SetProperty(SERVER_PROPERTIES::REBOOTING, 0);
Scalr::FireEvent($DBFarm->ID, new HostDownEvent($DBServer));
continue;
} elseif ($DBServer->GetRealStatus()->IsRunning() && $DBServer->status != SERVER_STATUS::RUNNING) {
if ($DBServer->status != SERVER_STATUS::TERMINATED) {
if ($DBServer->platform == SERVER_PLATFORMS::RDS) {
//TODO: timeouts
if ($DBServer->status == SERVER_STATUS::PENDING) {
$info = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
$event = new HostInitEvent($DBServer, $info['localIp'], $info['remoteIp'], '');
} elseif ($DBServer->status == SERVER_STATUS::INIT) {
$event = new HostUpEvent($DBServer, "");
// TODO: add mysql replication password
}
if ($event) {
Scalr::FireEvent($DBServer->farmId, $event);
} else {
//TODO: Log
}
} else {
if ($DBServer->platform == SERVER_PLATFORMS::NIMBULA) {
if (!$DBServer->GetProperty(NIMBULA_SERVER_PROPERTIES::USER_DATA_INJECTED)) {
$dbRole = $DBServer->GetFarmRoleObject()->GetRoleObject();
$ssh2Client = new Scalr_Net_Ssh2_Client();
$ssh2Client->addPassword($dbRole->getProperty(DBRole::PROPERTY_NIMBULA_INIT_ROOT_USER), $dbRole->getProperty(DBRole::PROPERTY_NIMBULA_INIT_ROOT_PASS));
$info = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
$port = $dbRole->getProperty(DBRole::PROPERTY_SSH_PORT);
if (!$port) {
$port = 22;
}
try {
$ssh2Client->connect($info['remoteIp'], $port);
foreach ($DBServer->GetCloudUserData() as $k => $v) {
$u_data .= "{$k}={$v};";
}
$u_data = trim($u_data, ";");
$ssh2Client->sendFile('/etc/scalr/private.d/.user-data', $u_data, "w+", false);
$DBServer->SetProperty(NIMBULA_SERVER_PROPERTIES::USER_DATA_INJECTED, 1);
} catch (Exception $e) {
Logger::getLogger(LOG_CATEGORY::FARM)->error(new FarmLogMessage($DBFarm->ID, $e->getMessage()));
}
}
}
try {
$dtadded = strtotime($DBServer->dateAdded);
$DBFarmRole = $DBServer->GetFarmRoleObject();
$launch_timeout = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) > 0 ? $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) : 300;
} catch (Exception $e) {
if (stristr($e->getMessage(), "not found")) {
PlatformFactory::NewPlatform($DBServer->platform)->TerminateServer($DBServer);
$DBServer->status = SERVER_STATUS::TERMINATED;
$DBServer->Save();
}
}
$scripting_event = false;
if ($DBServer->status == SERVER_STATUS::PENDING) {
$event = "hostInit";
$scripting_event = EVENT_TYPE::HOST_INIT;
} elseif ($DBServer->status == SERVER_STATUS::INIT) {
$event = "hostUp";
$scripting_event = EVENT_TYPE::HOST_UP;
}
if ($scripting_event) {
$scripting_timeout = (int) $this->db->GetOne("SELECT sum(timeout) FROM farm_role_scripts \n\t\t\t\t\t\t\t\t\t\tWHERE event_name=? AND \n\t\t\t\t\t\t\t\t\t\tfarm_roleid=? AND issync='1'", array($scripting_event, $DBServer->farmRoleId));
if ($scripting_timeout) {
$launch_timeout = $launch_timeout + $scripting_timeout;
}
if ($dtadded + $launch_timeout < time()) {
//Add entry to farm log
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, "Server '{$DBServer->serverId}' did not send '{$event}' event in {$launch_timeout} seconds after launch (Try increasing timeouts in role settings). Considering it broken. Terminating instance."));
try {
Scalr::FireEvent($DBFarm->ID, new BeforeHostTerminateEvent($DBServer, false));
} catch (Exception $err) {
$this->logger->fatal($err->getMessage());
}
}
}
// Is IP address changed?
if (!$DBServer->IsRebooting()) {
$ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
if ($ipaddresses['remoteIp'] && $DBServer->remoteIp != $ipaddresses['remoteIp']) {
Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp']));
}
//TODO: Check health:
}
}
}
} elseif ($DBServer->GetRealStatus()->isRunning() && $DBServer->status == SERVER_STATUS::RUNNING) {
// Is IP address changed?
if (!$DBServer->IsRebooting()) {
$ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
if ($ipaddresses['remoteIp'] && $DBServer->remoteIp != $ipaddresses['remoteIp']) {
Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp']));
}
//TODO: Check health:
} else {
//TODO: Check reboot timeout
}
}
if ($DBServer->status == SERVER_STATUS::PENDING_TERMINATE || $DBServer->status == SERVER_STATUS::TERMINATED) {
if ($DBServer->status == SERVER_STATUS::TERMINATED || !$DBServer->dateShutdownScheduled || $DBServer->dateShutdownScheduled && strtotime($DBServer->dateShutdownScheduled) + 60 * 3 < time()) {
try {
if (!$DBServer->GetRealStatus()->isTerminated()) {
try {
if ($DBServer->GetFarmRoleObject()->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ)) {
$serversCount = count($DBServer->GetFarmRoleObject()->GetServersByFilter(array(), array('status' => SERVER_STATUS::TERMINATED)));
if ($DBServer->index == 1 && $serversCount > 1) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("RabbitMQ role. Main DISK node should be terminated after all other nodes. Waiting... (Platform: %s) (Poller).", $DBServer->serverId, $DBServer->platform)));
continue;
}
}
} catch (Exception $e) {
}
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Terminating server '%s' (Platform: %s) (Poller).", $DBServer->serverId, $DBServer->platform)));
PlatformFactory::NewPlatform($DBServer->platform)->TerminateServer($DBServer);
$this->db->Execute("UPDATE servers_history SET\n\t\t\t\t\t\t\t\t\t\tdtterminated\t= NOW(),\n\t\t\t\t\t\t\t\t\t\tterminate_reason\t= ?\n\t\t\t\t\t\t\t\t\t\tWHERE server_id = ?\n\t\t\t\t\t\t\t\t\t", array(sprintf("Server is running on Amazon but has terminated status on Scalr"), $DBServer->serverId));
}
} catch (Exception $e) {
if (stristr($e->getMessage(), "not found")) {
$this->db->Execute("UPDATE servers_history SET\n\t\t\t\t\t\t\t\t\t\tdtterminated\t= NOW(),\n\t\t\t\t\t\t\t\t\t\tterminate_reason\t= ?\n\t\t\t\t\t\t\t\t\t\tWHERE server_id = ?\n\t\t\t\t\t\t\t\t\t", array(sprintf("Role was removed from farm"), $DBServer->serverId));
$DBServer->Remove();
} elseif (stristr($e->getMessage(), "disableApiTermination")) {
continue;
} else {
throw $e;
}
}
}
}
} catch (Exception $e) {
if (stristr($e->getMessage(), "not found")) {
var_dump($e);
} else {
print "[Farm: {$farmId}] {$e->getMessage()} at {$e->getFile()}:{$e->getLine()}\n\n";
}
}
}
}
