public function xLockAction() { $this->request->defineParams(array('serverId')); $dbServer = DBServer::LoadByID($this->getParam('serverId')); $this->user->getPermissions()->validate($dbServer); if ($dbServer->platform != SERVER_PLATFORMS::EC2) { throw new Exception("Server lock supported ONLY by EC2"); } $env = Scalr_Environment::init()->loadById($dbServer->envId); $ec2 = $env->aws($dbServer->GetCloudLocation())->ec2; $newValue = !$ec2->instance->describeAttribute($dbServer->GetCloudServerID(), InstanceAttributeType::disableApiTermination()); $ec2->instance->modifyAttribute($dbServer->GetCloudServerID(), InstanceAttributeType::disableApiTermination(), $newValue); $dbServer->SetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED, $newValue); $this->response->success(); }
public function xLockAction() { $this->request->defineParams(array('serverId')); $dbServer = DBServer::LoadByID($this->getParam('serverId')); $this->user->getPermissions()->validate($dbServer); if ($dbServer->platform != SERVER_PLATFORMS::EC2) { throw new Exception("Server lock supported ONLY by EC2"); } $env = Scalr_Environment::init()->loadById($dbServer->envId); $ec2 = $env->aws($dbServer->GetCloudLocation())->ec2; if ($dbServer->GetRealStatus(true)->isTerminated()) { $dbServer->SetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED, 0); $this->response->warning('Server was terminated. Clearing disableAPITermination flag.'); } else { $newValue = !$ec2->instance->describeAttribute($dbServer->GetCloudServerID(), InstanceAttributeType::disableApiTermination()); $ec2->instance->modifyAttribute($dbServer->GetCloudServerID(), InstanceAttributeType::disableApiTermination(), $newValue); $dbServer->SetProperties([EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME => 0, EC2_SERVER_PROPERTIES::IS_LOCKED => $newValue]); $this->response->success(); } }
/** * @test */ public function testFunctionalEc2() { $this->skipIfEc2PlatformDisabled(); $aws = $this->getContainer()->aws(AwsTestCase::REGION); $aws->ec2->enableEntityManager(); //We should use different ec2 instance for another region $aws2 = $this->getContainer()->aws(Aws::REGION_US_WEST_2); $nameTag = new ResourceTagSetData(self::TAG_NAME_KEY, self::getTestName(self::NAME_TAG_VALUE)); $subnetList = $aws->ec2->subnet->describe(); $this->assertInstanceOf($this->getEc2ClassName('DataType\\SubnetList'), $subnetList); unset($subnetList); $imageList = $aws->ec2->image->describe(null, 'self'); $this->assertInstanceOf($this->getEc2ClassName('DataType\\ImageList'), $imageList); unset($imageList); $volumeList = $aws->ec2->volume->describe(); $this->assertInstanceOf($this->getEc2ClassName('DataType\\VolumeList'), $volumeList); unset($volumeList); $snapshotList = $aws->ec2->snapshot->describe(null, 'self', new SnapshotFilterData(SnapshotFilterNameType::tag(self::TAG_NAME_KEY), self::getTestName(self::NAME_TAG_VALUE))); $this->assertInstanceOf($this->getEc2ClassName('DataType\\SnapshotList'), $snapshotList); /* @var $sn SnapshotList */ foreach ($snapshotList as $sn) { $sn->delete(); } unset($snapshotList); $reservationsList = $aws->ec2->instance->describe(null, new InstanceFilterData(InstanceFilterNameType::tag(self::TAG_NAME_KEY), self::getTestName(self::NAME_TAG_VALUE))); $this->assertInstanceOf($this->getEc2ClassName('DataType\\ReservationList'), $reservationsList); foreach ($reservationsList as $r) { /* @var $i InstanceData */ foreach ($r->instancesSet as $i) { $ds = $i->describeStatus(); if ($ds->instanceState->name == InstanceStateData::NAME_RUNNING) { //Whether the elastic ip is still associated with the instance $adlist = $aws->ec2->address->describe(null, null, array(array('name' => AddressFilterNameType::instanceId(), 'value' => $i->instanceId))); foreach ($adlist as $v) { //Removes associated elastic IP address $aws->ec2->address->disassociate($v->publicIp); $aws->ec2->address->release($v->publicIp); } unset($adlist); } else { if ($ds->instanceState->name == InstanceStateData::NAME_TERMINATED || $ds->instanceState->name == InstanceStateData::NAME_SHUTTING_DOWN) { continue; } } $i->terminate(); $i = $i->refresh(); for ($t = time(); time() - $t < 100 && isset($i->instanceState) && !in_array($i->instanceState->name, array(InstanceStateData::NAME_TERMINATED)); sleep(5)) { $i = $i->refresh(); } } } $placementGroups = $aws->ec2->placementGroup->describe(null, new PlacementGroupFilterData(PlacementGroupFilterNameType::groupName(), self::getTestName('placement-group'))); $this->assertInstanceOf($this->getEc2ClassName('DataType\\PlacementGroupList'), $placementGroups); /* @var $pg PlacementGroupData */ foreach ($placementGroups as $pg) { $pg->delete(); unset($pg); } unset($placementGroups); $reservedInstancesList = $aws->ec2->reservedInstance->describe(); $this->assertInstanceOf($this->getEc2ClassName('DataType\\ReservedInstanceList'), $reservedInstancesList); $availabilityZoneList = $aws->ec2->availabilityZone->describe('us-east-1a', array(array('name' => AvailabilityZoneFilterNameType::state(), 'value' => array('available')))); $this->assertInstanceOf($this->getEc2ClassName('DataType\\AvailabilityZoneList'), $availabilityZoneList); $this->assertEquals(1, count($availabilityZoneList)); $this->assertInstanceOf($this->getAwsClassName('Ec2'), $availabilityZoneList->getEc2()); $this->assertNotEmpty($availabilityZoneList->getRequestId()); $this->assertEquals('us-east-1a', $availabilityZoneList[0]->getZoneName()); $this->assertEquals('available', $availabilityZoneList[0]->getZoneState()); $this->assertEquals('us-east-1', $availabilityZoneList[0]->getRegionName()); $ml = $availabilityZoneList[0]->getMessageSet(); $this->assertInstanceOf($this->getEc2ClassName('DataType\\AvailabilityZoneMessageList'), $ml); $this->assertInstanceOf($this->getAwsClassName('Ec2'), $ml->getEc2()); unset($ml); $al = $aws->ec2->address->describe(null, null, array(array('name' => AddressFilterNameType::domain(), 'value' => 'standard'))); $this->assertInstanceOf($this->getEc2ClassName('DataType\\AddressList'), $al); unset($al); //Describes keypair $keyname = self::getTestName('keyname'); $kplist = $aws->ec2->keyPair->describe(null, array('key-name' => $keyname)); $this->assertInstanceOf($this->getEc2ClassName('DataType\\KeyPairList'), $kplist); if (count($kplist) > 0) { foreach ($kplist as $kp) { $kp->delete(); } } unset($kplist); //Creates keypair $kp = $aws->ec2->keyPair->create($keyname); $this->assertInstanceOf($this->getEc2ClassName('DataType\\KeyPairData'), $kp); $this->assertEquals($keyname, $kp->keyName); $this->assertNotNull($kp->keyFingerprint); $this->assertNotNull($kp->keyMaterial); //We should be assured that group which is used for the test does not exists $list = $aws->ec2->securityGroup->describe(null, null, new SecurityGroupFilterData(SecurityGroupFilterNameType::groupName(), self::getTestName('security-group'))); if (count($list) > 0) { foreach ($list as $v) { $v->delete(); } } unset($list); //Creates security group $securityGroupId = $aws->ec2->securityGroup->create(self::getTestName('security-group'), self::getTestName('security-group') . ' description'); $this->assertNotEmpty($securityGroupId); sleep(2); /* @var $sg SecurityGroupData */ $sg = $aws->ec2->securityGroup->describe(null, $securityGroupId)->get(0); $this->assertInstanceOf($this->getEc2ClassName('DataType\\SecurityGroupData'), $sg); $this->assertNotEmpty($sg->groupId); $this->assertEquals(self::getTestName('security-group'), $sg->groupName); $this->assertContains(self::getTestName('security-group'), $sg->groupDescription); $ret = $sg->createTags($nameTag); $this->assertTrue($ret); //Verifies that security group entity which is stored in manager is the same object $sgMirror = $aws->ec2->securityGroup->get($sg->groupId); $this->assertSame($sg, $sgMirror); unset($sgMirror); //Athorizes Security Group Ingress $ipperm = new IpPermissionData('tcp', 80, 80, new IpRangeList(array(new IpRangeData('192.0.2.0/24'), new IpRangeData('192.51.100.0/24')))); $ret = $sg->authorizeIngress($ipperm); $this->assertTrue($ret); $ipperm2 = new IpPermissionData('tcp', 8080, 8080, new IpRangeList(array(new IpRangeData('192.66.12.0/24')))); $ret = $sg->authorizeIngress($ipperm2); $this->assertTrue($ret); $ret = $sg->revokeIngress($ipperm2); $this->assertTrue($ret); //Describes itself $sg->refresh(); $this->assertContains(self::TAG_NAME_KEY, $sg->tagSet->getQueryArrayBare('Tag')); //Creates placement group $ret = $aws->ec2->placementGroup->create(self::getTestName('placement-group')); $this->assertTrue($ret); //Sometimes it takes a moment sleep(3); $pg = $aws->ec2->placementGroup->describe(self::getTestName('placement-group'))->get(0); $this->assertInstanceOf($this->getEc2ClassName('DataType\\PlacementGroupData'), $pg); $this->assertEquals(self::getTestName('placement-group'), $pg->groupName); $this->assertSame($pg, $aws->ec2->placementGroup->get($pg->groupName)); //RunInstance test $request = new RunInstancesRequestData(self::INSTANCE_IMAGE_ID, 1, 1); $request->instanceType = self::INSTANCE_TYPE; //Placement groups may not be used with instances of type 'm1.small'. $request->setPlacement(new PlacementResponseData(AwsTestCase::AVAILABILITY_ZONE_C)); $request->setMonitoring(true); $request->ebsOptimized = false; $request->userData = base64_encode("test=26;"); $request->appendSecurityGroupId($securityGroupId); $request->appendBlockDeviceMapping(new BlockDeviceMappingData("/dev/sdb", 'ephemeral0', null, new EbsBlockDeviceData(1, null, null, null, true, true))); $request->keyName = $keyname; $rd = $aws->ec2->instance->run($request); $this->assertInstanceOf($this->getEc2ClassName('DataType\\ReservationData'), $rd); /* @var $ind InstanceData */ $ind = $rd->instancesSet[0]; unset($request); //Monitors instance $ret = $ind->monitor(); $this->assertInstanceOf($this->getEc2ClassName('DataType\\MonitorInstancesResponseSetList'), $ret); $this->assertContains($ret->get(0)->monitoring->state, array(InstanceMonitoringStateData::STATE_PENDING, InstanceMonitoringStateData::STATE_ENABLED)); unset($ret); //Instance state must be in the running state for ($t = time(), $s = 5; time() - $t < 300 && $ind->instanceState->name !== InstanceStateData::NAME_RUNNING; $s += 5) { sleep($s); $ind = $ind->refresh(); } $this->assertEquals(InstanceStateData::NAME_RUNNING, $ind->instanceState->name); // route53 test $dns = self::getTestName('testname') . '.com'; // clean up $listZones = $aws->route53->zone->describe(); $this->assertInstanceOf($this->getRoute53ClassName('DataType\\ZoneList'), $listZones); /* @var $zone \Scalr\Service\Aws\Route53\DataType\ZoneData */ foreach ($listZones as $zone) { $this->assertNotNull($zone->zoneId); if ($dns . '.' == $zone->name) { $rrsRequest = new ChangeRecordSetsRequestData(); $rrsCnahgeList = new ChangeRecordSetList(); $rrsCnahgeListData = new ChangeRecordSetData('DELETE'); $rrsData = new RecordSetData($dns, 'A', 60); $rrsData->setResourceRecord([['value' => $ind->ipAddress]]); $rrsCnahgeListData->setRecordSet($rrsData); $rrsCnahgeList->append($rrsCnahgeListData); $rrsRequest->setChange($rrsCnahgeList); $changeRequest = $aws->route53->record->update($zone->zoneId, $rrsRequest); $this->assertInstanceOf($this->getRoute53ClassName('DataType\\ChangeData'), $changeRequest); $zone->delete($zone->zoneId); } unset($zone); } unset($listZones); $list = $aws->route53->health->describe(); $this->assertInstanceOf($this->getRoute53ClassName('DataType\\HealthList'), $list); foreach ($list as $check) { /* @var $check \Scalr\Service\Aws\Route53\DataType\HealthData */ $this->assertInstanceOf($this->getRoute53ClassName('DataType\\HealthConfigData'), $check->healthConfig); $this->assertNotNull($check->healthId); $this->assertNotNull($check->healthConfig->domainName); if ($dns . '.' == $check->healthConfig->domainName) { $check->delete($check->healthId); } unset($check); } unset($list); // create hosted zone $config = new ZoneData($dns); $hostedZone = $aws->route53->zone->create($config); $this->assertInstanceOf($this->getRoute53ClassName('DataType\\ZoneData'), $hostedZone); $this->assertNotNull($hostedZone->zoneId); $zoneId = (string) $hostedZone->zoneId; // create record set for hosted zone $rrsRequest = new ChangeRecordSetsRequestData(); $rrsCnahgeList = new ChangeRecordSetList(); $rrsCnahgeListData = new ChangeRecordSetData('CREATE'); $rrsData = new RecordSetData($dns, 'A', 60); $rrsData->setResourceRecord([['value' => $ind->ipAddress]]); $rrsCnahgeListData->setRecordSet($rrsData); $rrsCnahgeList->append($rrsCnahgeListData); $rrsRequest->setChange($rrsCnahgeList); $changeRequest = $aws->route53->record->update($zoneId, $rrsRequest); $this->assertInstanceOf($this->getRoute53ClassName('DataType\\ChangeData'), $changeRequest); $this->assertNotNull($changeRequest->changeId); $getChange = $aws->route53->record->fetch($changeRequest->changeId); $this->assertEquals($changeRequest->changeId, $getChange->changeId); // create health check using record set's idAddress $healthData = new HealthData(); $healthConfig = new HealthConfigData($ind->ipAddress, 80, 'HTTP', '/index.html', $dns); $healthData->setHealthConfig($healthConfig); $healthCheck = $aws->route53->health->create($healthData); $this->assertNotNull($healthCheck->healthId); $list = $aws->route53->health->describe(); $this->assertInstanceOf($this->getRoute53ClassName('DataType\\HealthList'), $list); foreach ($list as $check) { $this->assertInstanceOf($this->getRoute53ClassName('DataType\\HealthConfigData'), $check->healthConfig); $this->assertNotNull($check->healthId); $this->assertNotNull($check->healthConfig->domainName); if ($dns == $check->healthConfig->domainName) { $getCheck = $aws->route53->health->fetch($check->healthId); $this->assertEquals($check->healthId, $getCheck->healthId); $check->delete($check->healthId); } unset($check); } unset($list); // clean up $listZones = $aws->route53->zone->describe(); $this->assertInstanceOf($this->getRoute53ClassName('DataType\\ZoneList'), $listZones); foreach ($listZones as $zone) { /* @var $zone \Scalr\Service\Aws\Route53\DataType\ZoneData */ $this->assertNotNull($zone->zoneId); if ($dns . '.' == $zone->name) { $rrsRequest = new ChangeRecordSetsRequestData(); $rrsCnahgeList = new ChangeRecordSetList(); $rrsCnahgeListData = new ChangeRecordSetData('DELETE'); $rrsData = new RecordSetData($dns, 'A', 60); $rrsData->setResourceRecord([['value' => $ind->ipAddress]]); $rrsCnahgeListData->setRecordSet($rrsData); $rrsCnahgeList->append($rrsCnahgeListData); $rrsRequest->setChange($rrsCnahgeList); $changeRequest = $aws->route53->record->update($zone->zoneId, $rrsRequest); $this->assertInstanceOf($this->getRoute53ClassName('DataType\\ChangeData'), $changeRequest); $getZone = $aws->route53->zone->fetch($zone->zoneId); $this->assertEquals($zone->zoneId, $getZone->zoneId); $zone->delete($zone->zoneId); } unset($zone); } unset($listZones); //Creates the tag for the instance $res = $ind->createTags(array($nameTag, array('key' => 'Extratag', 'value' => 'extravalue'))); $this->assertTrue($res); //Verifies that tag has been successfully set. $ind->refresh(); $this->assertContains(self::TAG_NAME_KEY, $ind->tagSet->getQueryArrayBare()); $this->assertContains('Extratag', $ind->tagSet->getQueryArrayBare()); //Removes an extratag $ind->deleteTags(array(array('key' => 'Extratag', 'value' => null))); $this->assertEquals(self::INSTANCE_TYPE, $ind->instanceType); $this->assertEquals(self::INSTANCE_IMAGE_ID, $ind->imageId); $this->assertEquals(false, $ind->ebsOptimized); $this->assertContains($ind->monitoring->state, array('enabled', 'pending')); $this->assertEquals(AwsTestCase::AVAILABILITY_ZONE_C, $ind->placement->availabilityZone); $this->assertEquals($keyname, $ind->keyName); $this->assertContains($securityGroupId, array_map(function ($arr) { return $arr->groupId; }, iterator_to_array($ind->groupSet, false))); $this->assertContains(array("/dev/sdb", ''), array_map(function ($arr) { return array($arr->deviceName, $arr->virtualName); }, iterator_to_array($ind->blockDeviceMapping, false))); //Creates AMI // $cr = new CreateImageRequestData($ind->instanceId, sprintf(self::getTestName('i%s'), $ind->instanceId)); // $cr->description = 'It is supposed to be removed immediately after creation.'; // $imageId = $aws->ec2->image->create($cr); // sleep(3); // $ami = $aws->ec2->image->describe($imageId); // $this->assertInstanceOf($this->getEc2ClassName('DataType\\ImageData'), $ami); // $this->assertNotNull($ami->imageId); // //Waits while snapshot is created. // for ($t = time(), $s = 5; time() - $t < 300 && $ami->imageState === ImageData::STATE_PENDING; $s += 5) { // sleep($s); // $ami = $ami->refresh(); // } // $this->assertTrue(in_array($ami->imageState, array(ImageData::STATE_AVAILABLE, SnapshotData::STATUS_ERROR))); // unset($cr); // $ami->createTags($nameTag); // //Copy // $copyImageId = $ami->copy(substr(AwsTestCase::AVAILABILITY_ZONE_A, 0, -1), null, 'phpunit copied AMI', null, Aws::REGION_US_WEST_2); // $this->assertNotNull($copyImageId); // /* @var $ami2 ImageData */ // $ami2 = $aws2->ec2->image->describe($copyImageId)->get(0); // $this->assertNotNull($ami2); // $ami2->deregister(); // //Deregisters an AMI // $ami->deregister(); //Creates Elastic IP Address $address = $aws->ec2->address->allocate(); $this->assertInstanceOf($this->getEc2ClassName('DataType\\AddressData'), $address); $this->assertNotNull($address->publicIp); $ret = $aws->ec2->address->associate(new AssociateAddressRequestData($ind->instanceId, $address->publicIp)); $this->assertTrue($ret); //DescribeAddress should return allocated addres $adlist = $aws->ec2->address->describe($address->publicIp, null, array(array('name' => AddressFilterNameType::instanceId(), 'value' => $ind->instanceId))); $this->assertInstanceOf($this->getEc2ClassName('DataType\\AddressList'), $adlist); $this->assertEquals(1, count($adlist)); $this->assertEquals($address->publicIp, $adlist->get(0)->publicIp); //It must be associated with the instance $this->assertEquals($ind->instanceId, $adlist->get(0)->instanceId); $this->assertEquals($address->domain, $adlist->get(0)->domain); //Dissassociates address $ret = $aws->ec2->address->disassociate($address->publicIp); $this->assertTrue($ret); //Releases address $ret = $aws->ec2->address->release($address->publicIp); $this->assertTrue($ret); unset($adlist); unset($address); //Creates the volume $cvRequest = new CreateVolumeRequestData(AwsTestCase::AVAILABILITY_ZONE_C); $cvRequest->encrypted = true; $cvRequest->setSize(2)->setVolumeType(CreateVolumeRequestData::VOLUME_TYPE_STANDARD); $vd = $aws->ec2->volume->create($cvRequest); $this->assertInstanceOf($this->getEc2ClassName('DataType\\VolumeData'), $vd); $volumeid = $vd->volumeId; $this->assertTrue($vd->encrypted); unset($cvRequest); $res = $vd->createTags($nameTag); $this->assertTrue($res); //Volume must be in the Available status for ($t = time(), $s = 2; time() - $t < 300 && $vd->status !== VolumeData::STATUS_AVAILABLE; $s += 5) { sleep($s); $vd = $vd->refresh(); } $this->assertEquals(VolumeData::STATUS_AVAILABLE, $vd->status); //Attaching the volume $at = $vd->attach($ind->instanceId, '/dev/sdh'); $this->assertInstanceOf($this->getEc2ClassName('DataType\\AttachmentSetResponseData'), $at); //Creates snapshot $sn = $vd->createSnapshot(self::getTestName(self::NAME_SNAPSHOT)); $this->assertTrue($sn->encrypted); $this->assertInstanceOf($this->getEc2ClassName('DataType\\SnapshotData'), $sn); $this->assertNotEmpty($sn->snapshotId); $sn->createTags($nameTag); //Waits while snapshot is created. for ($t = time(), $s = 2; time() - $t < 300 && $sn->status === SnapshotData::STATUS_PENDING; $s += 5) { sleep($s); $sn->refresh(); } $this->assertTrue(in_array($sn->status, array(SnapshotData::STATUS_COMPLETED, SnapshotData::STATUS_ERROR))); //Copies snapshot to different region $copySnapshotId = $sn->copy($aws->getRegion(), 'phpunit copied encrypted snapshot', Aws::REGION_US_WEST_2, $aws->getPresignedUrl('ec2', 'CopySnapshot', Aws::REGION_US_WEST_2, $sn->snapshotId)); $this->assertNotNull($copySnapshotId); $aws2->ec2->tag->create($copySnapshotId, $nameTag); $csn = $aws2->ec2->snapshot->describe($copySnapshotId)->get(0); $this->assertInstanceOf($this->getEc2ClassName('DataType\\SnapshotData'), $csn); //Waits while snapshot is created. for ($t = time(), $s = 2; time() - $t < 600 && $csn->status === SnapshotData::STATUS_PENDING; $s += 5) { sleep($s); $csn = $csn->refresh(); } $this->assertTrue($csn->status == SnapshotData::STATUS_COMPLETED); $this->assertTrue($csn->encrypted); //Removes snapshot $ret = $sn->delete(); $this->assertTrue($ret); //Removes copied snapshot $ret = $csn->delete(); $this->assertTrue($ret); //Detaching the volume $at = $vd->detach(); $this->assertInstanceOf($this->getEc2ClassName('DataType\\AttachmentSetResponseData'), $at); $maxTimeout = 200; $interval = 2; while ($maxTimeout > 0) { if (count($vd->attachmentSet) == 0 || $vd->attachmentSet[0]->status == AttachmentSetResponseData::STATUS_DETACHED) { break; } sleep($interval); $maxTimeout -= $interval; $interval *= 2; $vd->refresh(); } if (isset($vd->attachmentSet[0]->status) && $vd->attachmentSet[0]->status !== AttachmentSetResponseData::STATUS_DETACHED) { $this->assertTrue(false, sprintf('The volume %s has not been detached from the instance %s yet.', $volumeid, $ind->instanceId)); } $vd->refresh(); $this->assertContains(self::TAG_NAME_KEY, $vd->tagSet->getQueryArrayBare('Tag')); //Deletes the volume. $ret = $vd->delete(); $this->assertTrue($ret); $this->assertNull($aws->ec2->volume->get($volumeid)); unset($volumeid); unset($vd); unset($at); $ind->refresh(); //Verifies that extratag has been successfully removed $this->assertNotContains('Extratag', $ind->tagSet->getQueryArrayBare()); $consoleOutput = $ind->getConsoleOutput(); $this->assertInstanceOf($this->getEc2ClassName('DataType\\GetConsoleOutputResponseData'), $consoleOutput); unset($consoleOutput); //Reboots the instance $ret = $ind->reboot(); $this->assertTrue($ret); //Stoping the instance $scList = $ind->stop(true); $this->assertInstanceOf($this->getEc2ClassName('DataType\\InstanceStateChangeList'), $scList); $this->assertEquals(1, count($scList)); unset($scList); for ($t = time(); time() - $t < 300 && $ind->instanceState->name !== InstanceStateData::NAME_STOPPED; sleep(5)) { $ind = $ind->refresh(); } $this->assertEquals(InstanceStateData::NAME_STOPPED, $ind->instanceState->name); //For an each attributes it will get it values foreach (InstanceAttributeType::getAllowedValues() as $attibute) { //This is allowed only for VPC instances, so we have to skip if ($attibute == InstanceAttributeType::TYPE_SOURCE_DEST_CHECK) { continue; } $attrValue = $ind->describeAttribute($attibute); } //Modifies instance attribute //Instance is required to be stopped. $ret = $ind->modifyAttribute(InstanceAttributeType::userData(), base64_encode('user data')); $this->assertTrue($ret); //Starts the instance $scList = $ind->start(); $this->assertInstanceOf($this->getEc2ClassName('DataType\\InstanceStateChangeList'), $scList); unset($scList); $ind = $ind->refresh(); for ($t = time(), $s = 5; time() - $t < 200 && $ind->instanceState->name !== InstanceStateData::NAME_RUNNING; $s += 5) { sleep($s); $ind = $ind->refresh(); } $this->assertEquals(InstanceStateData::NAME_RUNNING, $ind->instanceState->name); //Unmonitors instance $ret = $ind->unmonitor(); $this->assertInstanceOf($this->getEc2ClassName('DataType\\MonitorInstancesResponseSetList'), $ret); $this->assertContains($ret->get(0)->monitoring->state, array(InstanceMonitoringStateData::STATE_DISABLING, InstanceMonitoringStateData::STATE_DISABLED)); unset($ret); //Terminates the instance $st = $ind->terminate(); $this->assertInstanceOf($this->getEc2ClassName('DataType\\InstanceStateChangeList'), $st); $this->assertEquals(1, count($st)); $this->assertEquals($rd->instancesSet[0]->instanceId, $st[0]->getInstanceId()); for ($t = time(); time() - $t < 200 && $ind && $ind->instanceState->name != InstanceStateData::NAME_TERMINATED; sleep(5)) { $ind = $ind->refresh(); } $this->assertTrue(!$ind || $ind->instanceState->name == InstanceStateData::NAME_TERMINATED); if (isset($ind)) { unset($ind); } //Removes keypair $ret = $kp->delete(); $this->assertTrue($ret); unset($kp); //Removes security group $sg->delete(); //Verifies that security group is detached from the storage $sgMirror = $aws->ec2->securityGroup->get($sg->groupId); $this->assertNull($sgMirror); unset($sg); //Deletes placement group. $ret = $pg->delete(); $this->assertTrue($ret); $this->assertNull($aws->ec2->placementGroup->get(self::getTestName('placement-group'))); //Releases all memory $aws->getEntityManager()->detachAll(); }
/** * {@inheritdoc} * @see \Scalr\System\Zmq\Cron\TaskInterface::worker() */ public function worker($request) { $dtNow = new DateTime('now'); try { $dbServer = DBServer::LoadByID($request->serverId); } catch (ServerNotFoundException $e) { $this->log('INFO', "Server:%s does not exist:%s", $request->serverId, $e->getMessage()); return false; } if (!in_array($dbServer->status, [SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::PENDING_SUSPEND, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED])) { return false; } // Check and skip locked instances if ($dbServer->status == SERVER_STATUS::PENDING_TERMINATE && $dbServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED) == 1) { if (($checkDateTime = $dbServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME)) <= time()) { if (!$dbServer->GetRealStatus(true)->isTerminated()) { $isLocked = $dbServer->GetEnvironmentObject()->aws($dbServer->GetCloudLocation())->ec2->instance->describeAttribute($dbServer->GetCloudServerID(), InstanceAttributeType::disableApiTermination()); if ($isLocked) { \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($dbServer->GetFarmObject()->ID, sprintf("Server '%s' has disableAPITermination flag and can't be terminated (Platform: %s) (ServerTerminate).", $dbServer->serverId, $dbServer->platform), $dbServer->serverId)); $startTime = strtotime($dbServer->dateShutdownScheduled); // 1, 2, 3, 4, 5, 6, 9, 14, ... 60 $diff = round((($checkDateTime < $startTime ? $startTime : $checkDateTime) - $startTime) / 60 * 0.5) * 60; $diff = $diff == 0 ? 60 : ($diff > 3600 ? 3600 : $diff); $dbServer->SetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME, time() + $diff); return false; } else { $dbServer->SetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED, $isLocked); } } } else { return false; } } //Warming up static DI cache \Scalr::getContainer()->warmup(); // Reconfigure observers \Scalr::ReconfigureObservers(); if ($dbServer->status == SERVER_STATUS::TERMINATED || $dbServer->dateShutdownScheduled <= $dtNow->format('Y-m-d H:i:s')) { try { $p = PlatformFactory::NewPlatform($dbServer->platform); $environment = $dbServer->GetEnvironmentObject(); if (!$environment->isPlatformEnabled($dbServer->platform)) { throw new Exception(sprintf("%s platform is not enabled in the '%s' (%d) environment.", $dbServer->platform, $environment->name, $environment->id)); } if ($dbServer->GetCloudServerID()) { $serverHistory = $dbServer->getServerHistory(); $isTermination = in_array($dbServer->status, [SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE]); $isSuspension = in_array($dbServer->status, [SERVER_STATUS::SUSPENDED, SERVER_STATUS::PENDING_SUSPEND]); /* @var $terminationData TerminationData */ $terminationData = null; //NOTE: in any case, after call, be sure to set callback to null $this->setupClientCallback($p, function ($request, $response) use($dbServer, &$terminationData) { $terminationData = new TerminationData(); $terminationData->serverId = $dbServer->serverId; if ($request instanceof \http\Client\Request) { $terminationData->requestUrl = $request->getRequestUrl(); $terminationData->requestQuery = $request->getQuery(); $terminationData->request = $request->toString(); } if ($response instanceof \http\Client\Response) { $terminationData->response = $response->toString(); $terminationData->responseCode = $response->getResponseCode(); $terminationData->responseStatus = $response->getResponseStatus(); } }, $dbServer); try { $status = $dbServer->GetRealStatus(); } catch (Exception $e) { //eliminate callback $this->setupClientCallback($p, null, $dbServer); throw $e; } //eliminate callback $this->setupClientCallback($p, null, $dbServer); if ($dbServer->isCloudstack()) { //Workaround for when expunge flag not working and servers stuck in Destroyed state. $isTerminated = $status->isTerminated() && $status->getName() != 'Destroyed'; } else { $isTerminated = $status->isTerminated(); } if ($isTermination && !$isTerminated || $isSuspension && !$dbServer->GetRealStatus()->isSuspended()) { try { if ($dbServer->farmId != 0) { try { if ($dbServer->GetFarmRoleObject()->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ)) { $serversCount = count($dbServer->GetFarmRoleObject()->GetServersByFilter([], ['status' => [SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED]])); if ($dbServer->index == 1 && $serversCount > 1) { \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($dbServer->GetFarmObject()->ID, sprintf("RabbitMQ role. Main DISK node should be terminated after all other nodes. Waiting... (Platform: %s) (ServerTerminate).", $dbServer->serverId, $dbServer->platform), $dbServer->serverId)); return false; } } } catch (Exception $e) { } \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($dbServer->GetFarmObject()->ID, sprintf("Terminating server '%s' (Platform: %s) (ServerTerminate).", $dbServer->serverId, $dbServer->platform), $dbServer->serverId)); } } catch (Exception $e) { $this->getLogger()->warn("Server: %s caused exception: %s", $request->serverId, $e->getMessage()); } $terminationTime = $dbServer->GetProperty(SERVER_PROPERTIES::TERMINATION_REQUEST_UNIXTIME); if (!$terminationTime || time() - $terminationTime > 180) { if ($isTermination) { $p->TerminateServer($dbServer); } else { $p->SuspendServer($dbServer); } $dbServer->SetProperty(SERVER_PROPERTIES::TERMINATION_REQUEST_UNIXTIME, time()); if ($dbServer->farmId) { $wasHostDownFired = \Scalr::getDb()->GetOne("\n SELECT id FROM events WHERE event_server_id = ? AND type = ? AND is_suspend = '0'", array($request->serverId, 'HostDown')); if (!$wasHostDownFired) { $event = new HostDownEvent($dbServer); $event->isSuspended = !$isTermination; \Scalr::FireEvent($dbServer->farmId, $event); } } } } else { if ($dbServer->status == SERVER_STATUS::TERMINATED) { if (!$dbServer->dateShutdownScheduled || time() - strtotime($dbServer->dateShutdownScheduled) > 600) { $errorResolution = true; $serverHistory->setTerminated(); $dbServer->Remove(); if (isset($terminationData)) { $terminationData->save(); } } } else { if ($dbServer->status == SERVER_STATUS::PENDING_TERMINATE) { $dbServer->updateStatus(SERVER_STATUS::TERMINATED); $errorResolution = true; } else { if ($dbServer->status == SERVER_STATUS::PENDING_SUSPEND) { $dbServer->update(['status' => SERVER_STATUS::SUSPENDED, 'remoteIp' => '', 'localIp' => '']); $errorResolution = true; } } } if (!empty($errorResolution) && $request->attempts > 0 && ($ste = ServerTerminationError::findPk($request->serverId))) { //Automatic error resolution $ste->delete(); } } } else { //If there is no cloudserverID we don't need to add this server into server history. //$serverHistory->setTerminated(); $dbServer->Remove(); } } catch (InstanceNotFound $e) { if ($serverHistory) { $serverHistory->setTerminated(); } $dbServer->Remove(); if (isset($terminationData)) { $terminationData->save(); } } catch (Exception $e) { if ($request->serverId && stripos($e->getMessage(), "modify its 'disableApiTermination' instance attribute and try again") !== false && $dbServer && $dbServer->platform == \SERVER_PLATFORMS::EC2) { // server has disableApiTermination flag on cloud, update server properties $dbServer->SetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED, 1); } else { if ($request->serverId && ($e instanceof InvalidCloudCredentialsException || CloudPlatformSuspensionInfo::isSuspensionException($e) || stripos($e->getMessage(), "tenant is disabled") !== false || stripos($e->getMessage(), "was not able to validate the provided access credentials") !== false || stripos($e->getMessage(), "platform is not enabled") !== false || stripos($e->getMessage(), "neither api key nor password was provided for the openstack config") !== false || stripos($e->getMessage(), "refreshing the OAuth2 token") !== false || strpos($e->getMessage(), "Cannot obtain endpoint url. Unavailable service") !== false)) { //Postpones unsuccessful task for 30 minutes. $ste = new ServerTerminationError($request->serverId, isset($request->attempts) ? $request->attempts + 1 : 1, $e->getMessage()); $minutes = rand(30, 40); $ste->retryAfter = new \DateTime('+' . $minutes . ' minutes'); if ($ste->attempts > self::MAX_ATTEMPTS && in_array($dbServer->status, [SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::TERMINATED])) { //We are going to remove those with Pending terminate status from Scalr after 1 month of unsuccessful attempts $dbServer->Remove(); if (isset($terminationData)) { $terminationData->save(); } } $ste->save(); } else { $this->log('ERROR', "Server:%s, failed - Exeption:%s %s", $request->serverId, get_class($e), $e->getMessage()); throw $e; } } } } return $request; }
/** * {@inheritdoc} * @see Scalr_System_Cronjob_MultiProcess_DefaultWorker::handleWork() */ function handleWork($farmId) { $this->cleanup(); $DBFarm = DBFarm::LoadByID($farmId); $account = Scalr_Account::init()->loadById($DBFarm->ClientID); $payAsYouGoTime = $account->getSetting(Scalr_Account::SETTING_BILLING_PAY_AS_YOU_GO_DATE); $GLOBALS["SUB_TRANSACTIONID"] = abs(crc32(posix_getpid() . $farmId)); $GLOBALS["LOGGER_FARMID"] = $farmId; $this->logger->info("[" . $GLOBALS["SUB_TRANSACTIONID"] . "] Begin polling farm (ID: {$DBFarm->ID}, Name: {$DBFarm->Name}, Status: {$DBFarm->Status})"); // Collect information from database $servers_count = $this->db->GetOne("\n SELECT COUNT(*) FROM servers WHERE farm_id = ? AND status NOT IN (?,?)\n ", array($DBFarm->ID, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED)); $this->logger->info("[FarmID: {$DBFarm->ID}] Found {$servers_count} farm instances in database"); if ($DBFarm->Status == FARM_STATUS::TERMINATED && $servers_count == 0) { return; } foreach ($DBFarm->GetServersByFilter(array(), array('status' => SERVER_STATUS::PENDING_LAUNCH)) as $DBServer) { /** @var DBServer $DBServer */ try { if ($DBServer->cloudLocation) { try { Logger::getLogger(LOG_CATEGORY::FARM)->info(sprintf("Initializing cloud cache: {$DBServer->cloudLocation} ({$DBServer->serverId})")); $p = PlatformFactory::NewPlatform($DBServer->platform); $list = $p->GetServersList($DBServer->GetEnvironmentObject(), $DBServer->cloudLocation); } catch (Exception $e) { Logger::getLogger(LOG_CATEGORY::FARM)->info(sprintf("Initializing cloud cache: FAILED")); } } if ($DBServer->status != SERVER_STATUS::PENDING && $DBServer->status != SERVER_STATUS::PENDING_TERMINATE) { if (!$p->IsServerExists($DBServer)) { try { $serverInfo = $p->GetServerExtendedInformation($DBServer); } catch (Exception $e) { Logger::getLogger(LOG_CATEGORY::FARM)->error(sprintf("[CRASH][FarmID: %d] Crash check for server '%s' failed: %s", $DBFarm->ID, $DBServer->serverId, $e->getMessage())); } if (!$serverInfo) { if ($p->debugLog) { Logger::getLogger('Openstack')->fatal($p->debugLog); } if (!in_array($DBServer->status, array(SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED))) { if ($DBServer->GetProperty(SERVER_PROPERTIES::CRASHED) == 1) { if (PlatformFactory::isOpenstack($DBServer->platform)) { $DBServer->SetProperty(SERVER_PROPERTIES::MISSING, 1); } else { $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED); Scalr::FireEvent($DBFarm->ID, new HostCrashEvent($DBServer)); } } else { $DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::CRASHED => 1, SERVER_PROPERTIES::MISSING => 1]); Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' found in database but not found on %s. Crashed.", $DBServer->serverId, $DBServer->platform))); } continue; } } else { Logger::getLogger(LOG_CATEGORY::FARM)->error(sprintf("[CRASH][FarmID: %d] False-positive crash check: %s (EnvID: %d)", $DBFarm->ID, $DBServer->serverId, $DBServer->envId)); } } else { $DBServer->SetProperty(SERVER_PROPERTIES::CRASHED, "0"); $DBServer->SetProperty(SERVER_PROPERTIES::MISSING, "0"); } } } catch (Exception $e) { if (stristr($e->getMessage(), "AWS was not able to validate the provided access credentials") || stristr($e->getMessage(), "Unable to sign AWS API request. Please, check your X.509")) { $env = Scalr_Environment::init()->LoadById($DBFarm->EnvID); $env->status = Scalr_Environment::STATUS_INACTIVE; $env->save(); $env->setPlatformConfig(array('system.auto-disable-reason' => $e->getMessage()), false); return; } if (stristr($e->getMessage(), "Could not connect to host")) { continue; } print "[0][Farm: {$farmId}] {$e->getMessage()} at {$e->getFile()}:{$e->getLine()}\n\n"; continue; } /* try { $realStatus = $DBServer->GetRealStatus()->getName(); if ($realStatus == 'stopped') { $DBServer->SetProperty(SERVER_PROPERTIES::SUB_STATUS, $realStatus); continue; } else { if ($DBServer->GetProperty(SERVER_PROPERTIES::SUB_STATUS) == 'stopped') $DBServer->SetProperty(SERVER_PROPERTIES::SUB_STATUS, ""); } } catch (Exception $e) {} */ try { if (!in_array($DBServer->status, array(SERVER_STATUS::SUSPENDED, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::PENDING_SUSPEND))) { $openstackErrorState = false; if (PlatformFactory::isOpenstack($DBServer->platform)) { if ($DBServer->GetRealStatus()->getName() === 'ERROR') { $openstackErrorState = true; } } if ($DBServer->GetRealStatus()->isTerminated() || $openstackErrorState) { Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) not running (Real state: %s, Scalr status: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName(), $DBServer->status))); $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED); $DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::RESUMING => 0]); Scalr::FireEvent($DBFarm->ID, new HostDownEvent($DBServer)); continue; } elseif ($DBServer->GetRealStatus()->isSuspended()) { // if server was suspended when it was running if ($DBServer->status == SERVER_STATUS::RUNNING) { Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) not running (Real state: %s, Scalr status: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName(), $DBServer->status))); $DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::RESUMING => 0, SERVER_PROPERTIES::SUB_STATUS => ""]); $DBServer->remoteIp = ""; $DBServer->localIp = ""; $DBServer->status = SERVER_STATUS::SUSPENDED; $DBServer->Save(); Scalr::FireEvent($DBFarm->ID, new HostDownEvent($DBServer)); continue; } else { // If server was suspended during initialization - we do not support this and need to terminate this instance $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED); continue; } } } if ($DBServer->status != SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->IsRunning()) { if ($DBServer->status == SERVER_STATUS::SUSPENDED) { //TODO: Depends on resume strategy need to set server status // For Openstack we need to re-accociate IPs $DBServer->SetProperty(\SERVER_PROPERTIES::RESUMING, 0); try { if ($DBServer->isOpenstack()) { $this->openstackSetFloatingIp($DBServer); } } catch (Exception $e) { if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) { $DBServer->SetProperties([SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => $e->getMessage()]); } } Scalr::FireEvent($DBFarm->ID, new HostUpEvent($DBServer, "")); continue; } elseif (!in_array($DBServer->status, array(SERVER_STATUS::TERMINATED, SERVER_STATUS::TROUBLESHOOTING))) { if ($DBServer->platform == SERVER_PLATFORMS::EC2) { if ($DBServer->status == SERVER_STATUS::PENDING && $DBFarm->GetSetting(DBFarm::SETTING_EC2_VPC_ID)) { if ($DBServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_AWS_VPC_INTERNET_ACCESS) != 'outbound-only') { $ipAddress = \Scalr\Modules\Platforms\Ec2\Helpers\EipHelper::setEipForServer($DBServer); if ($ipAddress) { $DBServer->remoteIp = $ipAddress; $DBServer->Save(); } } } } try { if ($DBServer->isOpenstack()) { $this->openstackSetFloatingIp($DBServer); } } catch (Exception $e) { if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) { $DBServer->SetProperties([SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => $e->getMessage()]); } } if ($DBServer->isCloudstack()) { if ($DBServer->status == SERVER_STATUS::PENDING) { $jobId = $DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::LAUNCH_JOB_ID); try { $cs = $DBServer->GetEnvironmentObject()->cloudstack($DBServer->platform); $res = $cs->queryAsyncJobResult($jobId); if ($res->jobstatus == 1) { $DBServer->SetProperties([CLOUDSTACK_SERVER_PROPERTIES::TMP_PASSWORD => $res->virtualmachine->password, CLOUDSTACK_SERVER_PROPERTIES::SERVER_NAME => $res->virtualmachine->name]); } //TODO: handle failed job: $res->jobresult->jobstatus == 2 } catch (Exception $e) { if ($DBServer->farmId) { Logger::getLogger("CloudStack")->error(new FarmLogMessage($DBServer->farmId, $e->getMessage())); } } } } try { $dtadded = strtotime($DBServer->dateAdded); $DBFarmRole = $DBServer->GetFarmRoleObject(); $launch_timeout = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) > 0 ? $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) : 900; } catch (Exception $e) { if (stristr($e->getMessage(), "not found")) { $DBServer->terminate(DBServer::TERMINATE_REASON_ROLE_REMOVED); } } $scripting_event = false; if ($DBServer->status == SERVER_STATUS::PENDING) { $event = "hostInit"; $scripting_event = EVENT_TYPE::HOST_INIT; } elseif ($DBServer->status == SERVER_STATUS::INIT) { $event = "hostUp"; $scripting_event = EVENT_TYPE::HOST_UP; } if ($scripting_event && $dtadded) { $scripting_timeout = (int) $this->db->GetOne("\n SELECT sum(timeout)\n FROM farm_role_scripts\n WHERE event_name=? AND\n farm_roleid=? AND issync='1'\n ", array($scripting_event, $DBServer->farmRoleId)); if ($scripting_timeout) { $launch_timeout = $launch_timeout + $scripting_timeout; } if ($dtadded + $launch_timeout < time() && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) { //Add entry to farm log Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' did not send '%s' event in %s seconds after launch (Try increasing timeouts in role settings). Considering it broken. Terminating instance.", $DBServer->serverId, $event, $launch_timeout))); try { $DBServer->terminate(array(DBServer::TERMINATE_REASON_SERVER_DID_NOT_SEND_EVENT, $event, $launch_timeout), false); } catch (Exception $err) { $this->logger->fatal($err->getMessage()); } } elseif ($DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) { //DO NOT TERMINATE MONGODB INSTANCES BY TIMEOUT! IT'S NOT SAFE //THINK ABOUT WORKAROUND } } // Is IP address changed? if (!$DBServer->IsRebooting()) { $ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer); if ($ipaddresses['remoteIp'] && $DBServer->remoteIp && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp && $DBServer->localIp != $ipaddresses['localIp']) { Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("RemoteIP: %s (%s), LocalIp: %s (%s) (Poller).", $DBServer->remoteIp, $ipaddresses['remoteIp'], $DBServer->localIp, $ipaddresses['localIp']))); Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp'])); } //TODO: Check health: } } } elseif ($DBServer->status == SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->isRunning()) { // Is IP address changed? if (!$DBServer->IsRebooting()) { $ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer); if ($ipaddresses['remoteIp'] && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp != $ipaddresses['localIp']) { Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp'])); } if ($payAsYouGoTime) { $initTime = $DBServer->GetProperty(SERVER_PROPERTIES::INITIALIZED_TIME); if ($initTime < $payAsYouGoTime) { $initTime = $payAsYouGoTime; } $runningHours = ceil((time() - $initTime) / 3600); $scuUsed = $runningHours * Scalr_Billing::getSCUByInstanceType($DBServer->GetFlavor(), $DBServer->platform); $this->db->Execute("UPDATE servers_history SET scu_used = ?, scu_updated = 0 WHERE server_id = ?", array($scuUsed, $DBServer->serverId)); } if ($DBServer->platform == SERVER_PLATFORMS::EC2) { $env = Scalr_Environment::init()->loadById($DBServer->envId); $ec2 = $env->aws($DBServer->GetCloudLocation())->ec2; //TODO: $time = $DBServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME); if (!$time || time() < $time + 1200) { $isEnabled = $ec2->instance->describeAttribute($DBServer->GetCloudServerID(), InstanceAttributeType::disableApiTermination()); $DBServer->SetProperties([EC2_SERVER_PROPERTIES::IS_LOCKED => $isEnabled, EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME => time()]); } } } else { //TODO: Check reboot timeout } } } catch (Exception $e) { if (stristr($e->getMessage(), "not found")) { print $e->getTraceAsString() . "\n"; } elseif (stristr($e->getMessage(), "Request limit exceeded")) { sleep(10); print "[1:SLEEP][Farm: {$farmId}] {$e->getMessage()} at {$e->getFile()}:{$e->getLine()}\n\n"; } else { print "[1][Farm: {$farmId}] {$e->getMessage()} at {$e->getFile()}:{$e->getLine()}\n\n"; } } } }
function handleWork($farmId) { $this->cleanup(); $DBFarm = DBFarm::LoadByID($farmId); $account = Scalr_Account::init()->loadById($DBFarm->ClientID); $payAsYouGoTime = $account->getSetting(Scalr_Account::SETTING_BILLING_PAY_AS_YOU_GO_DATE); $GLOBALS["SUB_TRANSACTIONID"] = abs(crc32(posix_getpid() . $farmId)); $GLOBALS["LOGGER_FARMID"] = $farmId; $this->logger->info("[" . $GLOBALS["SUB_TRANSACTIONID"] . "] Begin polling farm (ID: {$DBFarm->ID}, Name: {$DBFarm->Name}, Status: {$DBFarm->Status})"); // // Collect information from database // $servers_count = $this->db->GetOne("SELECT COUNT(*) FROM servers WHERE farm_id = ? AND status != ?", array($DBFarm->ID, SERVER_STATUS::TERMINATED)); $this->logger->info("[FarmID: {$DBFarm->ID}] Found {$servers_count} farm instances in database"); if ($DBFarm->Status == FARM_STATUS::TERMINATED && $servers_count == 0) { return; } foreach ($DBFarm->GetServersByFilter(array(), array('status' => SERVER_STATUS::PENDING_LAUNCH)) as $DBServer) { try { if ($DBServer->status != SERVER_STATUS::PENDING && $DBServer->status != SERVER_STATUS::PENDING_TERMINATE) { $p = PlatformFactory::NewPlatform($DBServer->platform); if (!$p->IsServerExists($DBServer)) { try { $serverInfo = $p->GetServerExtendedInformation($DBServer); } catch (Exception $e) { Logger::getLogger(LOG_CATEGORY::FARM)->error(sprintf("[CRASH][FarmID: {$DBFarm->ID}] Crash check for server '{$DBServer->serverId}' failed: {$e->getMessage()}")); } if (!$serverInfo) { if ($DBServer->status != SERVER_STATUS::PENDING_TERMINATE && $DBServer->status != SERVER_STATUS::TERMINATED) { if ($DBServer->GetProperty("system.crashed") == 1) { $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED); Scalr::FireEvent($DBFarm->ID, new HostCrashEvent($DBServer)); } else { $DBServer->SetProperty(SERVER_PROPERTIES::REBOOTING, 0); Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' found in database but not found on {$DBServer->platform}. Crashed.", $DBServer->serverId))); $DBServer->SetProperty("system.crashed", "1"); } continue; } } else { Logger::getLogger(LOG_CATEGORY::FARM)->error(sprintf("[CRASH][FarmID: {$DBFarm->ID}] False-positive crash check: {$DBServer->serverId}")); if ($DBServer->platform == SERVER_PLATFORMS::EC2) { Logger::getLogger(LOG_CATEGORY::FARM)->fatal(sprintf("[CRASH][FarmID: {$DBFarm->ID}] InstanceID: %s, List: %s", $DBServer->GetCloudServerID(), json_encode($p->instancesListCache))); } } } else { $DBServer->SetProperty("system.crashed", "0"); } } } catch (Exception $e) { if (stristr($e->getMessage(), "AWS was not able to validate the provided access credentials") || stristr($e->getMessage(), "Unable to sign AWS API request. Please, check your X.509")) { $env = Scalr_Environment::init()->LoadById($DBFarm->EnvID); $env->status = Scalr_Environment::STATUS_INACTIVE; $env->save(); $env->setPlatformConfig(array('system.auto-disable-reason' => $e->getMessage()), false); return; } if (stristr($e->getMessage(), "Could not connect to host")) { continue; } print "[0][Farm: {$farmId}] {$e->getMessage()} at {$e->getFile()}:{$e->getLine()}\n\n"; continue; } try { $realStatus = $DBServer->GetRealStatus()->getName(); if ($realStatus == 'stopped') { $DBServer->SetProperty(SERVER_PROPERTIES::SUB_STATUS, $realStatus); continue; } else { if ($DBServer->GetProperty(SERVER_PROPERTIES::SUB_STATUS) == 'stopped') { $DBServer->SetProperty(SERVER_PROPERTIES::SUB_STATUS, ""); } } } catch (Exception $e) { } try { if (!in_array($DBServer->status, array(SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE)) && $DBServer->GetRealStatus()->isTerminated()) { Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) not running (Real state: %s, Scalr status: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName(), $DBServer->status))); $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED); $DBServer->SetProperty(SERVER_PROPERTIES::REBOOTING, 0); Scalr::FireEvent($DBFarm->ID, new HostDownEvent($DBServer)); continue; } elseif ($DBServer->status != SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->IsRunning()) { if ($DBServer->status != SERVER_STATUS::TERMINATED && $DBServer->status != SERVER_STATUS::TROUBLESHOOTING) { /* if ($DBServer->platform == SERVER_PLATFORMS::NIMBULA) { if (!$DBServer->GetProperty(NIMBULA_SERVER_PROPERTIES::USER_DATA_INJECTED)) { $dbRole = $DBServer->GetFarmRoleObject()->GetRoleObject(); $ssh2Client = new Scalr_Net_Ssh2_Client(); $ssh2Client->addPassword( $dbRole->getProperty(DBRole::PROPERTY_NIMBULA_INIT_ROOT_USER), $dbRole->getProperty(DBRole::PROPERTY_NIMBULA_INIT_ROOT_PASS) ); $info = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer); $port = $dbRole->getProperty(DBRole::PROPERTY_SSH_PORT); if (!$port) $port = 22; try { $ssh2Client->connect($info['remoteIp'], $port); foreach ($DBServer->GetCloudUserData() as $k=>$v) $u_data .= "{$k}={$v};"; $u_data = trim($u_data, ";"); $ssh2Client->sendFile('/etc/scalr/private.d/.user-data', $u_data, "w+", false); $DBServer->SetProperty(NIMBULA_SERVER_PROPERTIES::USER_DATA_INJECTED, 1); } catch(Exception $e) { Logger::getLogger(LOG_CATEGORY::FARM)->error(new FarmLogMessage($DBFarm->ID, $e->getMessage())); } } } */ if ($DBServer->platform == SERVER_PLATFORMS::EC2) { if ($DBServer->status == SERVER_STATUS::PENDING && $DBFarm->GetSetting(DBFarm::SETTING_EC2_VPC_ID)) { if ($DBServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_AWS_VPC_INTERNET_ACCESS) != 'outbound-only') { $ipAddress = Modules_Platforms_Ec2_Helpers_Eip::setEipForServer($DBServer); if ($ipAddress) { $DBServer->remoteIp = $ipAddress; $DBServer->Save(); } } } } if ($DBServer->isOpenstack()) { $ipPool = $DBServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_OPENSTACK_IP_POOL); if (!$DBServer->remoteIp && ($DBServer->status == SERVER_STATUS::PENDING || $DBServer->status == SERVER_STATUS::INIT) && $ipPool) { $osClient = $DBServer->GetEnvironmentObject()->openstack($DBServer->platform, $DBServer->GetProperty(OPENSTACK_SERVER_PROPERTIES::CLOUD_LOCATION)); if ($osClient->hasService('network')) { $platform = PlatformFactory::NewPlatform($DBServer->platform); $serverIps = $platform->GetServerIPAddresses($DBServer); /********* USE Quantum (Neuron) NETWORK *******/ $ips = $osClient->network->floatingIps->list(); //Check free existing IP $ipAssigned = false; $ipAddress = false; $ipId = false; $ipInfo = false; foreach ($ips as $ip) { if ($ip->fixed_ip_address == $serverIps['localIp'] && $ip->port_id) { $ipAddress = $ip->floating_ip_address; $ipId = $ip->id; $ipAssigned = true; $ipInfo = $ip; break; } /* if (!$ip->fixed_ip_address && !$ipAddress) { $ipAddress = $ip->floating_ip_address; $ipId = $ip->id; $ipInfo = $ip; } */ } if ($ipInfo) { Logger::getLogger("Openstack")->warn(new FarmLogMessage($DBServer->farmId, "Found free floating IP: {$ipAddress} for use (" . json_encode($ipInfo) . ")")); } if (!$ipAssigned) { // Get instance port $ports = $osClient->network->ports->list(); foreach ($ports as $port) { if ($port->device_id == $DBServer->GetProperty(OPENSTACK_SERVER_PROPERTIES::SERVER_ID)) { $serverNetworkPort = $port->id; break; } } if (!$serverNetworkPort) { Logger::getLogger("Openstack")->error(new FarmLogMessage($DBServer->farmId, "Unable to identify network port of instance")); } else { if (!$ipAddress) { $publicNetworkId = $ipPool; /* $networks = $osClient->network->listNetworks(); foreach ($networks as $network) { if ($network->{"router:external"} == true) { $publicNetworkId = $network->id; } } if (!$publicNetworkId) { Logger::getLogger("Openstack")->error(new FarmLogMessage($DBServer->farmId, "Unable to identify public network to allocate" )); } else { */ $ip = $osClient->network->floatingIps->create($publicNetworkId, $serverNetworkPort); $ipAddress = $ip->floating_ip_address; $DBServer->SetProperties(array(OPENSTACK_SERVER_PROPERTIES::FLOATING_IP => $ip->floating_ip_address, OPENSTACK_SERVER_PROPERTIES::FLOATING_IP_ID => $ip->id)); Logger::getLogger("Openstack")->warn(new FarmLogMessage($DBServer->farmId, "Allocated new IP {$ipAddress} for port: {$serverNetworkPort}")); //} } else { /* Logger::getLogger("Openstack")->warn(new FarmLogMessage($DBServer->farmId, "Updating IP {$ipAddress} ({$ipId}) with port: {$serverNetworkPort}" )); $osClient->network->floatingIps->update($ipId, $serverNetworkPort); */ } } } else { Logger::getLogger("Openstack")->warn(new FarmLogMessage($DBServer->farmId, "IP: {$ipAddress} already assigned")); } } else { /********* USE NOVA NETWORK *******/ //Check free existing IP $ipAssigned = false; $ipAddress = false; $ips = $osClient->servers->floatingIps->list($ipPool); foreach ($ips as $ip) { //if (!$ip->instance_id) { // $ipAddress = $ip->ip; //} if ($ip->instance_id == $DBServer->GetProperty(OPENSTACK_SERVER_PROPERTIES::SERVER_ID)) { $ipAddress = $ip->ip; $ipAssigned = true; } } //If no free IP allocate new from pool if (!$ipAddress) { $ip = $osClient->servers->floatingIps->create($ipPool); $ipAddress = $ip->ip; $DBServer->SetProperties(array(OPENSTACK_SERVER_PROPERTIES::FLOATING_IP => $ip->ip, OPENSTACK_SERVER_PROPERTIES::FLOATING_IP_ID => $ip->id)); } if (!$ipAssigned) { //Associate floating IP with Instance $osClient->servers->addFloatingIp($DBServer->GetCloudServerID(), $ipAddress); } } if ($ipAddress) { $DBServer->remoteIp = $ipAddress; $DBServer->Save(); $DBServer->SetProperty(SERVER_PROPERTIES::SYSTEM_IGNORE_INBOUND_MESSAGES, null); } } } if ($DBServer->isCloudstack()) { if ($DBServer->status == SERVER_STATUS::PENDING) { $jobId = $DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::LAUNCH_JOB_ID); try { $platform = PlatformFactory::NewPlatform($DBServer->platform); $cs = Scalr_Service_Cloud_Cloudstack::newCloudstack($platform->getConfigVariable(Modules_Platforms_Cloudstack::API_URL, $DBServer->GetEnvironmentObject()), $platform->getConfigVariable(Modules_Platforms_Cloudstack::API_KEY, $DBServer->GetEnvironmentObject()), $platform->getConfigVariable(Modules_Platforms_Cloudstack::SECRET_KEY, $DBServer->GetEnvironmentObject()), $DBServer->platform); $res = $cs->queryAsyncJobResult($jobId); if ($res->jobresult->jobstatus == 1) { $DBServer->SetProperty(CLOUDSTACK_SERVER_PROPERTIES::TMP_PASSWORD, $res->jobresult->virtualmachine->password); $DBServer->SetProperty(CLOUDSTACK_SERVER_PROPERTIES::SERVER_NAME, $res->jobresult->virtualmachine->name); } //TODO: handle failed job: $res->jobresult->jobstatus == 2 } catch (Exception $e) { if ($DBServer->farmId) { Logger::getLogger("CloudStack")->error(new FarmLogMessage($DBServer->farmId, $e->getMessage())); } } } } try { $dtadded = strtotime($DBServer->dateAdded); $DBFarmRole = $DBServer->GetFarmRoleObject(); $launch_timeout = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) > 0 ? $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) : 900; } catch (Exception $e) { if (stristr($e->getMessage(), "not found")) { $DBServer->terminate('FARM_ROLE_NOT_FOUND'); } } $scripting_event = false; if ($DBServer->status == SERVER_STATUS::PENDING) { $event = "hostInit"; $scripting_event = EVENT_TYPE::HOST_INIT; } elseif ($DBServer->status == SERVER_STATUS::INIT) { $event = "hostUp"; $scripting_event = EVENT_TYPE::HOST_UP; } if ($scripting_event && $dtadded) { $scripting_timeout = (int) $this->db->GetOne("\n SELECT sum(timeout)\n FROM farm_role_scripts\n WHERE event_name=? AND\n farm_roleid=? AND issync='1'\n ", array($scripting_event, $DBServer->farmRoleId)); if ($scripting_timeout) { $launch_timeout = $launch_timeout + $scripting_timeout; } if ($dtadded + $launch_timeout < time() && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) { //Add entry to farm log Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' did not send '%s' event in %s seconds after launch (Try increasing timeouts in role settings). Considering it broken. Terminating instance.", $DBServer->serverId, $event, $launch_timeout))); try { $DBServer->terminate(array('SERVER_DID_NOT_SEND_EVENT', $event, $launch_timeout), false); } catch (Exception $err) { $this->logger->fatal($err->getMessage()); } } elseif ($DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) { //DO NOT TERMINATE MONGODB INSTANCES BY TIMEOUT! IT'S NOT SAFE //THINK ABOUT WORKAROUND } } // Is IP address changed? if (!$DBServer->IsRebooting()) { $ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer); if ($ipaddresses['remoteIp'] && $DBServer->remoteIp && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp && $DBServer->localIp != $ipaddresses['localIp']) { Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("RemoteIP: %s (%s), LocalIp: %s (%s) (Poller).", $DBServer->remoteIp, $ipaddresses['remoteIp'], $DBServer->localIp, $ipaddresses['localIp']))); Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp'])); } //TODO: Check health: } } } elseif ($DBServer->status == SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->isRunning()) { // Is IP address changed? if (!$DBServer->IsRebooting()) { $ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer); if ($ipaddresses['remoteIp'] && $DBServer->remoteIp && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp && $DBServer->localIp != $ipaddresses['localIp']) { Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp'])); } if ($payAsYouGoTime) { $initTime = $DBServer->GetProperty(SERVER_PROPERTIES::INITIALIZED_TIME); if ($initTime < $payAsYouGoTime) { $initTime = $payAsYouGoTime; } $runningHours = ceil((time() - $initTime) / 3600); $scuUsed = $runningHours * Scalr_Billing::getSCUByInstanceType($DBServer->GetFlavor()); $this->db->Execute("UPDATE servers_history SET scu_used = ?, scu_updated = 0 WHERE server_id = ?", array($scuUsed, $DBServer->serverId)); } //Update GCE ServerID if ($DBServer->platform == SERVER_PLATFORMS::GCE) { if ($DBServer->GetProperty(GCE_SERVER_PROPERTIES::SERVER_ID) == $DBServer->serverId) { $info = PlatformFactory::NewPlatform($DBServer->platform)->GetServerExtendedInformation($DBServer); $DBServer->SetProperty(GCE_SERVER_PROPERTIES::SERVER_ID, $info['Cloud Server ID']); } } if ($DBServer->platform == SERVER_PLATFORMS::EC2) { $env = Scalr_Environment::init()->loadById($DBServer->envId); $ec2 = $env->aws($DBServer->GetCloudLocation())->ec2; //TODO: $isEnabled = $ec2->instance->describeAttribute($DBServer->GetCloudServerID(), InstanceAttributeType::disableApiTermination()); $DBServer->SetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED, $isEnabled); } } else { //TODO: Check reboot timeout } } } catch (Exception $e) { if (stristr($e->getMessage(), "not found")) { print $e->getTraceAsString() . "\n"; } else { print "[1][Farm: {$farmId}] {$e->getMessage()} at {$e->getFile()}:{$e->getLine()}\n\n"; } } } }
/** * {@inheritdoc} * @see \Scalr\System\Zmq\Cron\TaskInterface::worker() */ public function worker($request) { $db = \Scalr::getDb(); //Speed up poller if ($this->config()->daemon) { //Warming up static DI cache \Scalr::getContainer()->warmup(); } // Reconfigure observers \Scalr::ReconfigureObservers(); $DBFarm = DBFarm::LoadByID($request->farmId); $account = Scalr_Account::init()->loadById($DBFarm->ClientID); $payAsYouGoTime = $account->getSetting(Scalr_Account::SETTING_BILLING_PAY_AS_YOU_GO_DATE); $transactionId = abs(crc32(posix_getpid() . $request->farmId)); $this->getLogger()->info("[%s] Begin polling farm (ID: %d, Name: %s, Status: %s)", $transactionId, $DBFarm->ID, $DBFarm->Name, $DBFarm->Status); //Retrieves the number of either terminated or suspended servers for the farm $servers_count = $db->GetOne("\n SELECT COUNT(*) AS cnt FROM servers WHERE farm_id = ? AND status NOT IN (?,?)\n ", [$DBFarm->ID, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED]); if ($DBFarm->Status == FARM_STATUS::TERMINATED && $servers_count == 0) { //There are no servers for this farm return; } $this->getLogger()->info("%d server%s for the farm: %d", $servers_count, $servers_count == 1 ? '' : 's', $DBFarm->ID); $config = \Scalr::getContainer()->config; foreach ($DBFarm->GetServersByFilter(array(), array('status' => SERVER_STATUS::PENDING_LAUNCH)) as $DBServer) { /* @var $DBServer \DBServer */ try { if ($DBServer->cloudLocation) { try { $this->getLogger()->info("Retrieving the list of the instances for %s, server: %s", $DBServer->cloudLocation, $DBServer->serverId); $p = PlatformFactory::NewPlatform($DBServer->platform); $p->GetServersList($DBServer->GetEnvironmentObject(), $DBServer->cloudLocation); } catch (Exception $e) { $this->getLogger()->error("[Server: %s] Could not retrieve the list of the instances: %s", $DBServer->serverId, $e->getMessage()); continue; } } if ($DBServer->status != SERVER_STATUS::PENDING && $DBServer->status != SERVER_STATUS::PENDING_TERMINATE) { if (!$p->IsServerExists($DBServer)) { try { $serverInfo = $p->GetServerExtendedInformation($DBServer); } catch (Exception $e) { $this->getLogger()->error("[CRASH][FarmID: %d] Crash check for server '%s' failed: %s", $DBFarm->ID, $DBServer->serverId, $e->getMessage()); } if (!$serverInfo) { if (!in_array($DBServer->status, [SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::TERMINATED])) { if ($DBServer->isOpenstack() && $DBServer->status == SERVER_STATUS::SUSPENDED) { continue; } elseif ($DBServer->platform == \SERVER_PLATFORMS::GCE && $DBServer->status == SERVER_STATUS::SUSPENDED) { $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED); Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' was terminated", $DBServer->serverId), $DBServer->serverId)); continue; } if ($DBServer->GetProperty(SERVER_PROPERTIES::CRASHED) == 1) { $action = 'terminate'; if ($config->defined("scalr.{$DBServer->platform}.action_on_missing_server")) { $action = $config->get("scalr.{$DBServer->platform}.action_on_missing_server"); } if ($action == 'flag') { $DBServer->SetProperty(SERVER_PROPERTIES::MISSING, 1); } else { $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED); \Scalr::FireEvent($DBFarm->ID, new HostCrashEvent($DBServer)); } } else { $DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::CRASHED => 1, SERVER_PROPERTIES::MISSING => 1]); Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' found in database but not found on %s. Crashed.", $DBServer->serverId, $DBServer->platform), $DBServer->serverId)); } continue; } } else { //http.persistent.handles.limit must be set to 0 for pecl-http version 1 $this->getLogger()->error("[CRASH][FarmID: %d] False-positive crash check: %s (EnvID: %d). Please verify current scalr install with app/www/testenvironment.php", $DBFarm->ID, $DBServer->serverId, $DBServer->envId); //More debug $this->getLogger()->error(sprintf("[CRASH][FarmID: %d] Debug: %s", $DBFarm->ID, json_encode($p->instancesListCache))); $this->getLogger()->error(sprintf("[CRASH][FarmID: %d] {$DBServer->GetCloudServerID()}")); } } else { $DBServer->SetProperties([SERVER_PROPERTIES::CRASHED => 0, SERVER_PROPERTIES::MISSING => 0]); } } } catch (Exception $e) { if (stristr($e->getMessage(), "AWS was not able to validate the provided access credentials") || stristr($e->getMessage(), "You are not authorized to perform this operation") || stristr($e->getMessage(), "Unable to sign AWS API request. Please, check your X.509")) { /* @var $env \Scalr_Environment */ $env = Scalr_Environment::init()->LoadById($DBFarm->EnvID); $env->status = Scalr_Environment::STATUS_INACTIVE; $env->save(); //Saving the reason why this environment is disabled $env->setPlatformConfig(['system.auto-disable-reason' => $e->getMessage()]); return; } elseif (stristr($e->getMessage(), "Could not connect to host")) { continue; } $this->getLogger()->warn("Exception for farm: %d with the message: %s, in the %s:%s", $request->farmId, $e->getMessage(), $e->getFile(), $e->getLine()); continue; } try { if (!in_array($DBServer->status, [SERVER_STATUS::SUSPENDED, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::PENDING_SUSPEND])) { $openstackErrorState = false; if (PlatformFactory::isOpenstack($DBServer->platform) && $DBServer->GetRealStatus()->getName() === 'ERROR') { $openstackErrorState = true; } if ($DBServer->GetRealStatus()->isTerminated() || $openstackErrorState) { if ($openstackErrorState) { try { $info = PlatformFactory::NewPlatform($DBServer->platform)->GetServerExtendedInformation($DBServer); Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) is not running. Status: %s. Terminating.", $DBServer->serverId, $DBServer->platform, $info['Status']), $DBServer->serverId)); } catch (Exception $e) { } } else { Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) is not running (Real state: %s, Scalr status: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName(), $DBServer->status), $DBServer->serverId)); } $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED); $DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::RESUMING => 0]); \Scalr::FireEvent($DBFarm->ID, new HostDownEvent($DBServer)); continue; } elseif ($DBServer->GetRealStatus()->isSuspended()) { //In case the server was suspended when it was running if ($DBServer->status == SERVER_STATUS::RUNNING) { Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) is not running (Status in cloud: %s, Status in Scalr: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName(), $DBServer->status), $DBServer->serverId)); $DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::RESUMING => 0]); $DBServer->remoteIp = ""; $DBServer->localIp = ""; $DBServer->status = SERVER_STATUS::SUSPENDED; $DBServer->Save(); $event = new HostDownEvent($DBServer); $event->isSuspended = true; \Scalr::FireEvent($DBFarm->ID, $event); continue; } else { //If the server was suspended during initialization //we do not support this and need to terminate this instance $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED); continue; } } } if ($DBServer->status != SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->IsRunning()) { if ($DBServer->status == SERVER_STATUS::SUSPENDED) { $platform = PlatformFactory::NewPlatform($DBServer->platform); if ($DBServer->platform == \SERVER_PLATFORMS::GCE) { if ($platform->GetServerRealStatus($DBServer)->getName() == 'STOPPING') { continue; } } // For Openstack we need to re-accociate IPs try { if ($DBServer->isOpenstack()) { $this->openstackSetFloatingIp($DBServer); } } catch (Exception $e) { if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) { $DBServer->SetProperties([SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => "Scalr is unable to allocate/associate floating IP with server: " . $e->getMessage()]); } } if ($DBServer->platform == \SERVER_PLATFORMS::CLOUDSTACK) { if (!$DBServer->remoteIp) { $DBServer->remoteIp = CloudstackHelper::getSharedIP($DBServer); $DBServer->Save(); } } if ($platform->getResumeStrategy() == \Scalr_Role_Behavior::RESUME_STRATEGY_INIT) { $DBServer->status = \SERVER_STATUS::PENDING; $DBServer->SetProperty(\SERVER_PROPERTIES::RESUMING, 1); $DBServer->dateAdded = date("Y-m-d H:i:s"); $DBServer->Save(); } else { $DBServer->SetProperty(\SERVER_PROPERTIES::RESUMING, 0); \Scalr::FireEvent($DBFarm->ID, new HostUpEvent($DBServer, "")); } continue; } elseif (!in_array($DBServer->status, array(SERVER_STATUS::TERMINATED, SERVER_STATUS::TROUBLESHOOTING))) { if ($DBServer->platform == SERVER_PLATFORMS::EC2) { if ($DBServer->status == SERVER_STATUS::PENDING) { if (!$DBServer->remoteIp && !$DBServer->localIp) { $ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer); if ($ipaddresses['remoteIp'] && !$DBServer->remoteIp || $ipaddresses['localIp'] && !$DBServer->localIp) { $DBServer->remoteIp = $ipaddresses['remoteIp']; $DBServer->localIp = $ipaddresses['localIp']; $DBServer->Save(); } //Add tags Ec2Helper::createServerTags($DBServer); } if ($DBFarm->GetSetting(DBFarm::SETTING_EC2_VPC_ID)) { if ($DBServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_AWS_VPC_INTERNET_ACCESS) != 'outbound-only') { $ipAddress = Ec2EipHelper::setEipForServer($DBServer); if ($ipAddress) { $DBServer->remoteIp = $ipAddress; $DBServer->Save(); } } } } } try { if ($DBServer->isOpenstack()) { $this->openstackSetFloatingIp($DBServer); } } catch (Exception $e) { if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) { $DBServer->SetProperties([SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => "Scalr is unable to allocate/associate floating IP with server:" . $e->getMessage()]); } } if ($DBServer->isCloudstack()) { if ($DBServer->status == SERVER_STATUS::PENDING) { $jobId = $DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::LAUNCH_JOB_ID); try { $cs = $DBServer->GetEnvironmentObject()->cloudstack($DBServer->platform); $res = $cs->queryAsyncJobResult($jobId); if ($res->jobstatus == 1) { $DBServer->SetProperties([CLOUDSTACK_SERVER_PROPERTIES::TMP_PASSWORD => $res->virtualmachine->password, CLOUDSTACK_SERVER_PROPERTIES::SERVER_NAME => $res->virtualmachine->name]); } //TODO handle failed job: $res->jobresult->jobstatus == 2 } catch (Exception $e) { if ($DBServer->farmId) { Logger::getLogger("CloudStack")->error(new FarmLogMessage($DBServer->farmId, $e->getMessage(), $DBServer->serverId)); } } } } try { $dtadded = strtotime($DBServer->dateAdded); $DBFarmRole = $DBServer->GetFarmRoleObject(); $launch_timeout = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) > 0 ? $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) : 900; } catch (Exception $e) { if (stristr($e->getMessage(), "not found")) { $DBServer->terminate(DBServer::TERMINATE_REASON_ROLE_REMOVED); } } $scripting_event = false; if ($DBServer->status == SERVER_STATUS::PENDING) { $event = "hostInit"; $scripting_event = EVENT_TYPE::HOST_INIT; } elseif ($DBServer->status == SERVER_STATUS::INIT) { $event = "hostUp"; $scripting_event = EVENT_TYPE::HOST_UP; } if ($scripting_event && $dtadded) { $scripting_timeout = (int) $db->GetOne("\n SELECT SUM(timeout)\n FROM farm_role_scripts\n WHERE event_name = ? AND farm_roleid = ? AND issync = '1'\n ", [$scripting_event, $DBServer->farmRoleId]); if ($scripting_timeout) { $launch_timeout = $launch_timeout + $scripting_timeout; } if ($dtadded + $launch_timeout < time() && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) { //Add entry to farm log Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' did not send '%s' event in %s seconds after launch (Try increasing timeouts in role settings). Considering it broken. Terminating instance.", $DBServer->serverId, $event, $launch_timeout), $DBServer->serverId)); try { $DBServer->terminate(array(DBServer::TERMINATE_REASON_SERVER_DID_NOT_SEND_EVENT, $event, $launch_timeout), false); } catch (Exception $err) { $this->getLogger()->fatal($err->getMessage()); } } elseif ($DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) { //DO NOT TERMINATE MONGODB INSTANCES BY TIMEOUT! IT'S NOT SAFE //THINK ABOUT WORKAROUND } } //Whether IP address is changed if (!$DBServer->IsRebooting()) { $ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer); if ($ipaddresses['remoteIp'] && $DBServer->remoteIp && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp && $DBServer->localIp != $ipaddresses['localIp']) { Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("RemoteIP: %s (%s), LocalIp: %s (%s) (Poller).", $DBServer->remoteIp, $ipaddresses['remoteIp'], $DBServer->localIp, $ipaddresses['localIp']), $DBServer->serverId)); \Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp'])); } //TODO Check health } } } elseif ($DBServer->status == SERVER_STATUS::SUSPENDED && $DBServer->GetRealStatus()->isTerminated()) { if ($DBServer->platform == SERVER_PLATFORMS::EC2) { $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED); \Scalr::FireEvent($DBFarm->ID, new HostCrashEvent($DBServer)); } elseif ($DBServer->platform == SERVER_PLATFORMS::GCE) { \Scalr::FireEvent($DBFarm->ID, new HostDownEvent($DBServer)); } } elseif ($DBServer->status == SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->isRunning()) { // Is IP address changed? if (!$DBServer->IsRebooting()) { $ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer); if ($ipaddresses['remoteIp'] && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp != $ipaddresses['localIp']) { \Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp'])); } if ($payAsYouGoTime) { $initTime = $DBServer->GetProperty(SERVER_PROPERTIES::INITIALIZED_TIME); if ($initTime < $payAsYouGoTime) { $initTime = $payAsYouGoTime; } $runningHours = ceil((time() - $initTime) / 3600); $scuUsed = $runningHours * Scalr_Billing::getSCUByInstanceType($DBServer->GetFlavor(), $DBServer->platform); $db->Execute("UPDATE servers_history SET scu_used = ?, scu_updated = 0 WHERE server_id = ?", [$scuUsed, $DBServer->serverId]); } if ($DBServer->platform == SERVER_PLATFORMS::EC2) { $env = Scalr_Environment::init()->loadById($DBServer->envId); $ec2 = $env->aws($DBServer->GetCloudLocation())->ec2; $time = $DBServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME); if (!$time || time() < $time + 1200) { $isEnabled = $ec2->instance->describeAttribute($DBServer->GetCloudServerID(), InstanceAttributeType::disableApiTermination()); $DBServer->SetProperties([EC2_SERVER_PROPERTIES::IS_LOCKED => $isEnabled, EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME => time()]); } } } else { //TODO Check reboot timeout } } } catch (Exception $e) { if (stristr($e->getMessage(), "not found")) { $this->getLogger()->fatal($e->getMessage()); } elseif (stristr($e->getMessage(), "Request limit exceeded")) { sleep(5); $this->getLogger()->error("[Farm: %d] sleep due to exception: %s", $request->farmId, $e->getMessage()); } else { $this->getLogger()->error("[Farm: %d] Exception: %s", $request->farmId, $e->getMessage()); } } } return $request; }
/** * {@inheritdoc} * @see \Scalr\System\Zmq\Cron\TaskInterface::worker() */ public function worker($request) { $db = \Scalr::getDb(); //The list of the suspension information about cloud platforms $this->aSuspensionInfo = []; //Speed up poller if ($this->config()->daemon) { //Warming up static DI cache \Scalr::getContainer()->warmup(); } // Reconfigure observers \Scalr::ReconfigureObservers(); $DBFarm = DBFarm::LoadByID($request->farmId); $account = Scalr_Account::init()->loadById($DBFarm->ClientID); $payAsYouGoTime = $account->getSetting(Scalr_Account::SETTING_BILLING_PAY_AS_YOU_GO_DATE); $transactionId = abs(crc32(posix_getpid() . $request->farmId)); $this->getLogger()->info("[%s] Begin polling farm (ID: %d, Name: %s, Status: %s, Platform:%s)", $transactionId, $DBFarm->ID, $DBFarm->Name, $DBFarm->Status, $request->platform); $jobStartTime = microtime(true); //Retrieves the number of either terminated or suspended servers for the farm $servers_count = $db->GetOne("\n SELECT COUNT(*) AS cnt FROM servers\n WHERE farm_id = ? AND platform = ? AND status NOT IN (?,?)\n ", [$DBFarm->ID, $request->platform, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED]); if ($DBFarm->Status == FARM_STATUS::TERMINATED && $servers_count == 0) { //There are no servers for this farm and platform return; } $this->getLogger()->info("%d server%s for the farm: %d and platform: %s", $servers_count, $servers_count == 1 ? '' : 's', $DBFarm->ID, $request->platform); $config = \Scalr::getContainer()->config; /* if ($request->platform) { $p = PlatformFactory::NewPlatform($request->platform); $p->ClearCache(); } */ $p = PlatformFactory::NewPlatform($request->platform); foreach ($DBFarm->GetServersByFilter(['platform' => $request->platform], ['status' => SERVER_STATUS::PENDING_LAUNCH]) as $DBServer) { /* @var $DBServer \DBServer */ //Get platform suspension info $suspensionInfo = $this->getSuspensionInfo($DBServer->platform, $DBServer->envId); //If the cloud platform is suspended we should not process it if ($suspensionInfo->isSuspended()) { continue; } try { //1. We need to check that server is exists in cloud and not missed. // (On Openstack server can be missed and should not be terminated) $cacheKey = sprintf('%s:%s', $DBServer->envId, $DBServer->cloudLocation); if ($DBServer->cloudLocation && count($p->instancesListCache[$cacheKey]) == 0) { try { $this->getLogger()->info("Retrieving the list of the instances for %s, server: %s, platform: %s", $DBServer->cloudLocation, $DBServer->serverId, $request->platform); if ($DBServer->platform == \SERVER_PLATFORMS::AZURE) { //For Azure we need to pass resource group instead of cloudLocation $p->GetServersList($DBServer->GetEnvironmentObject(), $DBServer->GetProperty(\AZURE_SERVER_PROPERTIES::RESOURCE_GROUP)); } else { $p->GetServersList($DBServer->GetEnvironmentObject(), $DBServer->cloudLocation); } //We successfully polled cloud so can resume suspension status for the cloud platform if ($suspensionInfo->isPendingSuspend()) { $suspensionInfo->resume(); } } catch (Exception $e) { if (CloudPlatformSuspensionInfo::isSuspensionException($e)) { $suspensionInfo->registerError($e->getMessage()); } $this->getLogger()->error("[Server: %s] Could not retrieve the list of the instances: %s", $DBServer->serverId, $e->getMessage()); continue; } } if ($DBServer->status != SERVER_STATUS::PENDING && $DBServer->status != SERVER_STATUS::PENDING_TERMINATE) { if (!$p->IsServerExists($DBServer)) { try { $serverInfo = $p->GetServerExtendedInformation($DBServer); } catch (Exception $e) { $this->getLogger()->error("[CRASH][FarmID: %d] Crash check for server '%s' failed: %s", $DBFarm->ID, $DBServer->serverId, $e->getMessage()); continue; } if (!$serverInfo) { if (!in_array($DBServer->status, [SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::TERMINATED])) { if ($DBServer->isOpenstack() && $DBServer->status == SERVER_STATUS::SUSPENDED) { continue; } elseif ($DBServer->platform == \SERVER_PLATFORMS::GCE && $DBServer->status == SERVER_STATUS::SUSPENDED) { $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED); \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf(_("Server '%s' was terminated"), $DBServer->serverId), $DBServer->serverId)); continue; } $action = 'terminate'; if ($config->defined("scalr.{$DBServer->platform}.action_on_missing_server")) { $action = $config->get("scalr.{$DBServer->platform}.action_on_missing_server"); } if ($action == 'flag' && !$DBServer->GetProperty(SERVER_PROPERTIES::MISSING)) { \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' found in Scalr but not found in the cloud (%s). Marking as Missing.", $DBServer->serverId, $DBServer->platform), $DBServer->serverId)); $DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::MISSING => 1]); } else { \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' found in Scalr but not found in the cloud (%s). Terminating.", $DBServer->serverId, $DBServer->platform), $DBServer->serverId)); $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED); } continue; } } else { //http.persistent.handles.limit must be set to 0 for pecl-http version 1 $this->getLogger()->error("[CRASH][FarmID: %d] False-positive crash check: %s (EnvID: %d). Please verify current scalr install with app/www/testenvironment.php", $DBFarm->ID, $DBServer->serverId, $DBServer->envId); } } else { $DBServer->SetProperties([SERVER_PROPERTIES::MISSING => 0]); } } } catch (Exception $e) { if (CloudPlatformSuspensionInfo::isSuspensionException($e)) { $suspensionInfo->registerError($e->getMessage()); } $this->getLogger()->warn("Exception for Farm: %d, Platform: %s with the message: %s, in the %s:%s", $request->farmId, $request->platform, $e->getMessage(), $e->getFile(), $e->getLine()); continue; } try { if (!in_array($DBServer->status, [SERVER_STATUS::SUSPENDED, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::PENDING_SUSPEND])) { $openstackErrorState = false; if (PlatformFactory::isOpenstack($DBServer->platform) && $DBServer->GetRealStatus()->getName() === 'ERROR') { $openstackErrorState = true; } if ($DBServer->GetRealStatus()->isTerminated() || $openstackErrorState) { // If openstack server is in ERROR state we need more details if ($openstackErrorState) { try { $info = $p->GetServerExtendedInformation($DBServer); $status = empty($info['Status']) ? false : $info['Status']; } catch (Exception $e) { } } if (empty($status)) { $status = $DBServer->GetRealStatus()->getName(); } \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) was terminated in cloud or from within an OS. Status: %s.", $DBServer->serverId, $DBServer->platform, $status), $DBServer->serverId)); $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED); continue; } elseif ($DBServer->GetRealStatus()->isSuspended()) { //In case the server was suspended when it was running if ($DBServer->status == SERVER_STATUS::RUNNING) { \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) is not running (Status in cloud: %s, Status in Scalr: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName(), $DBServer->status), $DBServer->serverId)); $event = new HostDownEvent($DBServer); $event->isSuspended = true; \Scalr::FireEvent($DBFarm->ID, $event); continue; } else { if ($DBServer->status != \SERVER_STATUS::RESUMING) { //If the server was suspended during initialization //we do not support this and need to terminate this instance if ($DBServer->platform == \SERVER_PLATFORMS::EC2) { try { $info = $p->GetServerExtendedInformation($DBServer); $realStatus = !empty($info['Instance state']) ? $info['Instance state'] : ''; } catch (\Exception $e) { // no need to do anything here; } $this->getLogger()->error("[SUSPEND_RESUME_ISSUE][ServerID: %s][2] Cached Cloud Status: %s (Cache age: %d seconds), Status: %s, Real status: %s", $DBServer->serverId, $DBServer->GetRealStatus()->getName(), time() - $p->instancesListCache[$cacheKey][$DBServer->GetCloudServerID()]['_timestamp'], $DBServer->status, $realStatus); } $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED); continue; } else { // Need to clear cache, because this situation happens only when cache is stale. $p->ClearCache(); } } } } if ($DBServer->status != SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->IsRunning()) { if ($DBServer->status == SERVER_STATUS::SUSPENDED) { if ($DBServer->platform == \SERVER_PLATFORMS::GCE) { if ($p->GetServerRealStatus($DBServer)->getName() == 'STOPPING') { continue; } } $update = []; // For Openstack we need to re-accociate IPs try { if ($DBServer->isOpenstack()) { OpenstackHelper::setServerFloatingIp($DBServer); } } catch (Exception $e) { if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) { $DBServer->SetProperties([\SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, \SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => "Scalr is unable to allocate/associate floating IP with server: " . $e->getMessage()]); } } if ($DBServer->platform == \SERVER_PLATFORMS::CLOUDSTACK) { if (!$DBServer->remoteIp) { $update['remoteIp'] = CloudstackHelper::getSharedIP($DBServer); } } if ($DBServer->platform == \SERVER_PLATFORMS::EC2) { try { $info = $p->GetServerExtendedInformation($DBServer); $realStatus = !empty($info['Instance state']) ? $info['Instance state'] : ''; } catch (\Exception $e) { // no need to do anything here; } $this->getLogger()->error("[SUSPEND_RESUME_ISSUE][ServerID: %s][1] Cached Cloud Status: %s (Cache age: %d seconds), Status: %s, Real status: %s", $DBServer->serverId, $DBServer->GetRealStatus()->getName(), time() - $p->instancesListCache[$cacheKey][$DBServer->GetCloudServerID()]['_timestamp'], $DBServer->status, $realStatus); } $update['status'] = \SERVER_STATUS::RESUMING; $update['dateAdded'] = date("Y-m-d H:i:s"); $DBServer->update($update); unset($update); continue; } elseif (!in_array($DBServer->status, array(SERVER_STATUS::TERMINATED))) { $elasticIpAssigned = false; if ($DBServer->platform == SERVER_PLATFORMS::EC2) { if ($DBServer->status == SERVER_STATUS::PENDING) { if (!$DBServer->remoteIp && !$DBServer->localIp) { $ipaddresses = $p->GetServerIPAddresses($DBServer); $elasticIpAddress = Ec2EipHelper::setEipForServer($DBServer); if ($elasticIpAddress) { $ipaddresses['remoteIp'] = $elasticIpAddress; $DBServer->remoteIp = $elasticIpAddress; $elasticIpAssigned = true; } if ($ipaddresses['remoteIp'] && !$DBServer->remoteIp || $ipaddresses['localIp'] && !$DBServer->localIp || $elasticIpAssigned) { $DBServer->update(['remoteIp' => $ipaddresses['remoteIp'], 'localIp' => $ipaddresses['localIp']]); } //Add tags Ec2Helper::createObjectTags($DBServer); } } } if ($DBServer->platform == \SERVER_PLATFORMS::AZURE) { if ($DBServer->GetProperty(\AZURE_SERVER_PROPERTIES::SZR_EXTENSION_DEPLOYED)) { if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) { // Check scalarizr deployment status $env = $DBServer->GetEnvironmentObject(); $azure = $env->azure(); $info = $azure->compute->virtualMachine->getInstanceViewInfo($env->cloudCredentials(SERVER_PLATFORMS::AZURE)->properties[Entity\CloudCredentialsProperty::AZURE_SUBSCRIPTION_ID], $DBServer->GetProperty(\AZURE_SERVER_PROPERTIES::RESOURCE_GROUP), $DBServer->GetProperty(\AZURE_SERVER_PROPERTIES::SERVER_NAME)); $extensions = !empty($info->extensions) ? $info->extensions : []; foreach ($extensions as $extension) { /* @var $extension ExtensionData */ if ($extension->name == 'scalarizr') { $extStatus = $extension->statuses[0]; /* @var $extStatus StatusData */ if ($extStatus->level == 'Error') { $DBServer->SetProperties([\SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, \SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => "Azure resource extension failed to provision scalr agent. Status: {$extStatus->code} ({$extStatus->message})"]); } } } } } else { AzureHelper::setupScalrAgent($DBServer); } } try { if ($DBServer->isOpenstack()) { OpenstackHelper::setServerFloatingIp($DBServer); } } catch (Exception $e) { if (!$DBServer->GetProperty(\SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) { $DBServer->SetProperties([\SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, \SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => "Scalr is unable to allocate/associate floating IP with server:" . $e->getMessage()]); } } if ($DBServer->isCloudstack()) { if ($DBServer->status == SERVER_STATUS::PENDING) { $jobId = $DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::LAUNCH_JOB_ID); try { $cs = $DBServer->GetEnvironmentObject()->cloudstack($DBServer->platform); $res = $cs->queryAsyncJobResult($jobId); if ($res->jobstatus == 1) { $DBServer->SetProperties([CLOUDSTACK_SERVER_PROPERTIES::TMP_PASSWORD => $res->virtualmachine->password, CLOUDSTACK_SERVER_PROPERTIES::SERVER_NAME => $res->virtualmachine->name]); } //TODO handle failed job: $res->jobresult->jobstatus == 2 } catch (Exception $e) { if ($DBServer->farmId) { \Scalr::getContainer()->logger("CloudStack")->error(new FarmLogMessage($DBServer->farmId, $e->getMessage(), $DBServer->serverId)); } } } } try { $dtadded = strtotime($DBServer->dateAdded); $DBFarmRole = $DBServer->GetFarmRoleObject(); $launchTimeout = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SYSTEM_LAUNCH_TIMEOUT) > 0 ? $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SYSTEM_LAUNCH_TIMEOUT) : 900; } catch (Exception $e) { if (stristr($e->getMessage(), "not found")) { $DBServer->terminate(DBServer::TERMINATE_REASON_ROLE_REMOVED); } } $scriptingEvent = false; $eventName = null; if ($DBServer->status == SERVER_STATUS::PENDING) { $eventName = "hostInit"; $scriptingEvent = EVENT_TYPE::HOST_INIT; } elseif ($DBServer->status == SERVER_STATUS::INIT) { $eventName = "hostUp"; $scriptingEvent = EVENT_TYPE::HOST_UP; } if ($scriptingEvent && $dtadded) { $hasPendingMessages = !!$db->GetOne("\n SELECT EXISTS(SELECT 1 FROM messages WHERE type='in' AND status='0' AND server_id = ?)\n ", [$DBServer->serverId]); $scriptingTimeout = (int) $db->GetOne("\n SELECT SUM(timeout)\n FROM farm_role_scripts\n WHERE event_name = ? AND farm_roleid = ? AND issync = '1'\n ", [$scriptingEvent, $DBServer->farmRoleId]); if ($scriptingTimeout) { $launchTimeout = $launchTimeout + $scriptingTimeout; } if (!$hasPendingMessages && $dtadded + $launchTimeout < time() && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) { //Add entry to farm log \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' did not send '%s' event in %s seconds after launch (Try increasing timeouts in role settings). Considering it broken. Terminating instance.", $DBServer->serverId, $eventName, $launchTimeout), $DBServer->serverId)); try { $DBServer->terminate(array(DBServer::TERMINATE_REASON_SERVER_DID_NOT_SEND_EVENT, $eventName, $launchTimeout), false); } catch (Exception $err) { $this->getLogger()->fatal($err->getMessage()); } } elseif ($DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) { //DO NOT TERMINATE MONGODB INSTANCES BY TIMEOUT! IT'S NOT SAFE //THINK ABOUT WORKAROUND } } //Whether IP address is changed if (!$DBServer->IsRebooting() && !$elasticIpAssigned) { $ipaddresses = $p->GetServerIPAddresses($DBServer); if ($ipaddresses['remoteIp'] && $DBServer->remoteIp && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp && $DBServer->localIp != $ipaddresses['localIp']) { \Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("RemoteIP: %s (%s), LocalIp: %s (%s) (Poller).", $DBServer->remoteIp, $ipaddresses['remoteIp'], $DBServer->localIp, $ipaddresses['localIp']), $DBServer->serverId)); \Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp'])); } //TODO Check health } } } elseif ($DBServer->status == SERVER_STATUS::SUSPENDED && $DBServer->GetRealStatus()->isTerminated()) { //TODO: Terminated outside scalr while in SUSPENDED state $DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED); } elseif ($DBServer->status == SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->isRunning()) { // Is IP address changed? if (!$DBServer->IsRebooting()) { $ipaddresses = $p->GetServerIPAddresses($DBServer); // Private IP cannot be removed (only changed). if ($DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp != $ipaddresses['localIp']) { \Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp'])); } if ($payAsYouGoTime) { $initTime = $DBServer->dateInitialized ? strtotime($DBServer->dateInitialized) : null; if ($initTime < $payAsYouGoTime) { $initTime = $payAsYouGoTime; } $runningHours = ceil((time() - $initTime) / 3600); $scuUsed = $runningHours * Scalr_Billing::getSCUByInstanceType($DBServer->getType(), $DBServer->platform); $db->Execute("UPDATE servers_history SET scu_used = ?, scu_updated = 0 WHERE server_id = ?", [$scuUsed, $DBServer->serverId]); } if ($DBServer->platform == SERVER_PLATFORMS::EC2) { $env = Scalr_Environment::init()->loadById($DBServer->envId); $ec2 = $env->aws($DBServer->GetCloudLocation())->ec2; $time = $DBServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME); if (!$time || time() > $time + 1200) { $isEnabled = $ec2->instance->describeAttribute($DBServer->GetCloudServerID(), InstanceAttributeType::disableApiTermination()); $DBServer->SetProperties([EC2_SERVER_PROPERTIES::IS_LOCKED => $isEnabled, EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME => time()]); } } } else { //TODO Check reboot timeout } } } catch (Exception $e) { if (CloudPlatformSuspensionInfo::isSuspensionException($e)) { $suspensionInfo->registerError($e->getMessage()); } if (stristr($e->getMessage(), "not found")) { $this->getLogger()->fatal($e->getMessage()); } elseif (stristr($e->getMessage(), "Request limit exceeded")) { sleep(5); $this->getLogger()->error("[Farm: %d] sleep due to exception: %s", $request->farmId, $e->getMessage()); } else { $this->getLogger()->error("[Farm: %d] Exception: %s", $request->farmId, $e->getMessage()); } } } $this->getLogger()->info("[%s] Finished farm polling (ID: %d, Name: %s, Status: %s, Platform:%s). Time: %s", $transactionId, $DBFarm->ID, $DBFarm->Name, $DBFarm->Status, $request->platform, microtime(true) - $jobStartTime); return $request; }