protected function executeAllocateResource(DrydockLease $lease)
{
$repository_id = $lease->getAttribute('repositoryID');
if (!$repository_id) {
throw new Exception("Lease is missing required 'repositoryID' attribute.");
}
// TODO: (T603) Figure out the interaction between policies and
// Drydock.
$repository = id(new PhabricatorRepository())->load($repository_id);
if (!$repository) {
throw new Exception("Repository '{$repository_id}' does not exist!");
}
switch ($repository->getVersionControlSystem()) {
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
break;
default:
throw new Exception('Unsupported VCS!');
}
// TODO: Policy stuff here too.
$host_lease = id(new DrydockLease())->setResourceType('host')->waitUntilActive();
$path = $host_lease->getAttribute('path') . $repository->getCallsign();
$this->log(pht('Cloning %s into %s....', $repository->getCallsign(), $path));
$cmd = $host_lease->getInterface('command');
$cmd->execx('git clone --origin origin %P %s', $repository->getRemoteURIEnvelope(), $path);
$this->log(pht('Complete.'));
$resource = $this->newResourceTemplate('Working Copy (' . $repository->getCallsign() . ')');
$resource->setStatus(DrydockResourceStatus::STATUS_OPEN);
$resource->setAttribute('lease.host', $host_lease->getID());
$resource->setAttribute('path', $path);
$resource->setAttribute('repositoryID', $repository->getID());
$resource->save();
return $resource;
}
public function loadPage()
{
$table = new DrydockLease();
$conn_r = $table->establishConnection('r');
$data = queryfx_all($conn_r, 'SELECT lease.* FROM %T lease %Q %Q %Q', $table->getTableName(), $this->buildWhereClause($conn_r), $this->buildOrderClause($conn_r), $this->buildLimitClause($conn_r));
return $table->loadAllFromArray($data);
}
protected function yieldIfExpiringLease(DrydockLease $lease)
{
if (!$lease->canReceiveCommands()) {
return;
}
$this->yieldIfExpiring($lease->getUntil());
}
public function activateLease(DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease)
{
$command_type = DrydockCommandInterface::INTERFACE_TYPE;
$interface = $lease->getInterface($command_type);
$cmd = array();
$arg = array();
$cmd[] = 'git clean -d --force';
$cmd[] = 'git reset --hard HEAD';
$cmd[] = 'git fetch';
$commit = $lease->getAttribute('commit');
$branch = $lease->getAttribute('branch');
if ($commit !== null) {
$cmd[] = 'git reset --hard %s';
$arg[] = $commit;
} else {
if ($branch !== null) {
$cmd[] = 'git reset --hard %s';
$arg[] = $branch;
}
}
$cmd = implode(' && ', $cmd);
$argv = array_merge(array($cmd), $arg);
$result = call_user_func_array(array($interface, 'execx'), $argv);
$lease->activateOnResource($resource);
}
public static function initializeNewLease()
{
$lease = new DrydockLease();
// Pregenerate a PHID so that the caller can set something up to release
// this lease before queueing it for activation.
$lease->setPHID($lease->generatePHID());
return $lease;
}
public function getInterface(DrydockResource $resource, DrydockLease $lease, $type)
{
switch ($type) {
case 'command':
return id(new DrydockSSHCommandInterface())->setConfiguration(array('host' => $resource->getAttribute('host'), 'port' => $resource->getAttribute('port'), 'credential' => $resource->getAttribute('credential'), 'platform' => $resource->getAttribute('platform')))->setWorkingDirectory($lease->getAttribute('path'));
case 'filesystem':
return id(new DrydockSFTPFilesystemInterface())->setConfiguration(array('host' => $resource->getAttribute('host'), 'port' => $resource->getAttribute('port'), 'credential' => $resource->getAttribute('credential')));
}
throw new Exception(pht("No interface of type '%s'.", $type));
}
public function getPendingLease()
{
if (!$this->lease) {
$lease = new DrydockLease();
$lease->setStatus(DrydockLeaseStatus::STATUS_PENDING);
$lease->save();
$this->lease = $lease;
}
return $lease;
}
public static function writeLog(DrydockResource $resource = null, DrydockLease $lease = null, $message)
{
$log = id(new DrydockLog())->setEpoch(time())->setMessage($message);
if ($resource) {
$log->setResourceID($resource->getID());
}
if ($lease) {
$log->setLeaseID($lease->getID());
}
$log->save();
}
public function getInterface(DrydockResource $resource, DrydockLease $lease, $type)
{
switch ($type) {
case 'webroot':
$iface = new DrydockApacheWebrootInterface();
$iface->setConfiguration(array('uri' => $lease->getAttribute('uri')));
return $iface;
case 'command':
$host_lease_id = $resource->getAttribute('lease.host');
$host_lease = id(new DrydockLease())->load($host_lease_id);
$host_lease->attachResource($host_lease->loadResource());
return $host_lease->getInterface($type);
}
throw new Exception("No interface of type '{$type}'.");
}
private function destroyLease(DrydockLease $lease)
{
$status = $lease->getStatus();
switch ($status) {
case DrydockLeaseStatus::STATUS_RELEASED:
case DrydockLeaseStatus::STATUS_BROKEN:
break;
default:
throw new PhabricatorWorkerPermanentFailureException(pht('Unable to destroy lease ("%s"), lease has the wrong ' . 'status ("%s").', $lease->getPHID(), $status));
}
$resource = $lease->getResource();
$blueprint = $resource->getBlueprint();
$blueprint->destroyLease($resource, $lease);
$lease->setStatus(DrydockLeaseStatus::STATUS_DESTROYED)->save();
}
private function releaseLease(DrydockLease $lease)
{
$lease->openTransaction();
$lease->setStatus(DrydockLeaseStatus::STATUS_RELEASED)->save();
// TODO: Hold slot locks until destruction?
DrydockSlotLock::releaseLocks($lease->getPHID());
$lease->saveTransaction();
PhabricatorWorker::scheduleTask('DrydockLeaseDestroyWorker', array('leasePHID' => $lease->getPHID()), array('objectPHID' => $lease->getPHID()));
$resource = $lease->getResource();
$blueprint = $resource->getBlueprint();
$blueprint->didReleaseLease($resource, $lease);
}
private function loadWorkingCopyLease(DrydockRepositoryOperation $operation)
{
$viewer = $this->getViewer();
// TODO: This is very similar to leasing in Harbormaster, maybe we can
// share some of the logic?
$lease_phid = $operation->getProperty('exec.leasePHID');
if ($lease_phid) {
$lease = id(new DrydockLeaseQuery())->setViewer($viewer)->withPHIDs(array($lease_phid))->executeOne();
if (!$lease) {
throw new PhabricatorWorkerPermanentFailureException(pht('Lease "%s" could not be loaded.', $lease_phid));
}
} else {
$working_copy_type = id(new DrydockWorkingCopyBlueprintImplementation())->getType();
$repository = $operation->getRepository();
$allowed_phids = $repository->getAutomationBlueprintPHIDs();
$authorizing_phid = $repository->getPHID();
$lease = DrydockLease::initializeNewLease()->setResourceType($working_copy_type)->setOwnerPHID($operation->getPHID())->setAuthorizingPHID($authorizing_phid)->setAllowedBlueprintPHIDs($allowed_phids);
$map = $this->buildRepositoryMap($operation);
$lease->setAttribute('repositories.map', $map);
$task_id = $this->getCurrentWorkerTaskID();
if ($task_id) {
$lease->setAwakenTaskIDs(array($task_id));
}
$operation->setProperty('exec.leasePHID', $lease->getPHID())->save();
$lease->queueForActivation();
}
if ($lease->isActivating()) {
throw new PhabricatorWorkerYieldException(15);
}
if (!$lease->isActive()) {
throw new PhabricatorWorkerPermanentFailureException(pht('Lease "%s" never activated.', $lease->getPHID()));
}
return $lease;
}
private function buildPropertyListView(DrydockLease $lease, PhabricatorActionListView $actions)
{
$viewer = $this->getViewer();
$view = new PHUIPropertyListView();
$view->setActionList($actions);
$view->addProperty(pht('Status'), DrydockLeaseStatus::getNameForStatus($lease->getStatus()));
$view->addProperty(pht('Resource Type'), $lease->getResourceType());
$resource_phid = $lease->getResourcePHID();
if ($resource_phid) {
$resource_display = $viewer->renderHandle($resource_phid);
} else {
$resource_display = phutil_tag('em', array(), pht('No Resource'));
}
$view->addProperty(pht('Resource'), $resource_display);
$until = $lease->getUntil();
if ($until) {
$until_display = phabricator_datetime($until, $viewer);
} else {
$until_display = phutil_tag('em', array(), pht('Never'));
}
$view->addProperty(pht('Expires'), $until_display);
$attributes = $lease->getAttributes();
if ($attributes) {
$view->addSectionHeader(pht('Attributes'), 'fa-list-ul');
foreach ($attributes as $key => $value) {
$view->addProperty($key, $value);
}
}
return $view;
}
private function buildPropertyListView(DrydockLease $lease, PhabricatorActionListView $actions)
{
$view = new PHUIPropertyListView();
$view->setActionList($actions);
switch ($lease->getStatus()) {
case DrydockLeaseStatus::STATUS_ACTIVE:
$status = pht('Active');
break;
case DrydockLeaseStatus::STATUS_RELEASED:
$status = pht('Released');
break;
case DrydockLeaseStatus::STATUS_EXPIRED:
$status = pht('Expired');
break;
case DrydockLeaseStatus::STATUS_PENDING:
$status = pht('Pending');
break;
case DrydockLeaseStatus::STATUS_BROKEN:
$status = pht('Broken');
break;
default:
$status = pht('Unknown');
break;
}
$view->addProperty(pht('Status'), $status);
$view->addProperty(pht('Resource Type'), $lease->getResourceType());
$view->addProperty(pht('Resource'), $lease->getResourceID());
$attributes = $lease->getAttributes();
if ($attributes) {
$view->addSectionHeader(pht('Attributes'));
foreach ($attributes as $key => $value) {
$view->addProperty($key, $value);
}
}
return $view;
}
public function execute(HarbormasterBuild $build, HarbormasterBuildTarget $build_target)
{
$viewer = PhabricatorUser::getOmnipotentUser();
$settings = $this->getSettings();
// TODO: We should probably have a separate temporary storage area for
// execution stuff that doesn't step on configuration state?
$lease_phid = $build_target->getDetail('exec.leasePHID');
if ($lease_phid) {
$lease = id(new DrydockLeaseQuery())->setViewer($viewer)->withPHIDs(array($lease_phid))->executeOne();
if (!$lease) {
throw new PhabricatorWorkerPermanentFailureException(pht('Lease "%s" could not be loaded.', $lease_phid));
}
} else {
$working_copy_type = id(new DrydockWorkingCopyBlueprintImplementation())->getType();
$allowed_phids = $build_target->getFieldValue('blueprintPHIDs');
if (!is_array($allowed_phids)) {
$allowed_phids = array();
}
$authorizing_phid = $build_target->getBuildStep()->getPHID();
$lease = DrydockLease::initializeNewLease()->setResourceType($working_copy_type)->setOwnerPHID($build_target->getPHID())->setAuthorizingPHID($authorizing_phid)->setAllowedBlueprintPHIDs($allowed_phids);
$map = $this->buildRepositoryMap($build_target);
$lease->setAttribute('repositories.map', $map);
$task_id = $this->getCurrentWorkerTaskID();
if ($task_id) {
$lease->setAwakenTaskIDs(array($task_id));
}
// TODO: Maybe add a method to mark artifacts like this as pending?
// Create the artifact now so that the lease is always disposed of, even
// if this target is aborted.
$build_target->createArtifact($viewer, $settings['name'], HarbormasterWorkingCopyArtifact::ARTIFACTCONST, array('drydockLeasePHID' => $lease->getPHID()));
$lease->queueForActivation();
$build_target->setDetail('exec.leasePHID', $lease->getPHID())->save();
}
if ($lease->isActivating()) {
// TODO: Smart backoff?
throw new PhabricatorWorkerYieldException(15);
}
if (!$lease->isActive()) {
// TODO: We could just forget about this lease and retry?
throw new PhabricatorWorkerPermanentFailureException(pht('Lease "%s" never activated.', $lease->getPHID()));
}
}
public function acquireLease(DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease)
{
$lease->setActivateWhenAcquired(true)->needSlotLock($this->getLeaseSlotLock($resource))->acquireOnResource($resource);
}
public function getInterface(DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease, $type)
{
switch ($type) {
case DrydockCommandInterface::INTERFACE_TYPE:
$host_lease = $this->loadHostLease($resource);
$command_interface = $host_lease->getInterface($type);
$path = $lease->getAttribute('workingcopy.default');
$command_interface->setWorkingDirectory($path);
return $command_interface;
}
}
public function getCommandError(DrydockLease $lease)
{
$error = $lease->getAttribute('workingcopy.vcs.error');
if (!$error) {
return null;
} else {
return $error;
}
}
protected function requireActiveLease(DrydockLease $lease)
{
$lease_status = $lease->getStatus();
switch ($lease_status) {
case DrydockLeaseStatus::STATUS_PENDING:
case DrydockLeaseStatus::STATUS_ACQUIRED:
throw new PhabricatorWorkerYieldException(15);
case DrydockLeaseStatus::STATUS_ACTIVE:
return;
default:
throw new Exception(pht('Lease ("%s") is in bad state ("%s"), expected "%s".', $lease->getPHID(), $lease_status, DrydockLeaseStatus::STATUS_ACTIVE));
}
}
/**
* Make sure that a lease was really acquired properly.
*
* @param DrydockBlueprint Blueprint which created the resource.
* @param DrydockResource Resource which was acquired.
* @param DrydockLease The lease which was supposedly acquired.
* @return void
* @task lease
*/
private function validateAcquiredLease(DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease)
{
if (!$lease->isAcquiredLease()) {
throw new Exception(pht('Blueprint "%s" (of type "%s") is not properly implemented: it ' . 'returned from "%s" without acquiring a lease.', $blueprint->getBlueprintName(), $blueprint->getClassName(), 'acquireLease()'));
}
$lease_phid = $lease->getResourcePHID();
$resource_phid = $resource->getPHID();
if ($lease_phid !== $resource_phid) {
// TODO: Destroy the lease?
throw new Exception(pht('Blueprint "%s" (of type "%s") is not properly implemented: it ' . 'returned from "%s" with a lease acquired on the wrong resource.', $blueprint->getBlueprintName(), $blueprint->getClassName(), 'acquireLease()'));
}
}
private function allocateLease(DrydockLease $lease)
{
$type = $lease->getResourceType();
$blueprints = $this->loadAllBlueprints();
// TODO: Policy stuff.
$pool = id(new DrydockResource())->loadAllWhere('type = %s AND status = %s', $lease->getResourceType(), DrydockResourceStatus::STATUS_OPEN);
$this->logToDrydock(pht('Found %d Open Resource(s)', count($pool)));
$candidates = array();
foreach ($pool as $key => $candidate) {
if (!isset($blueprints[$candidate->getBlueprintPHID()])) {
unset($pool[$key]);
continue;
}
$blueprint = $blueprints[$candidate->getBlueprintPHID()];
$implementation = $blueprint->getImplementation();
if ($implementation->filterResource($candidate, $lease)) {
$candidates[] = $candidate;
}
}
$this->logToDrydock(pht('%d Open Resource(s) Remain', count($candidates)));
$resource = null;
if ($candidates) {
shuffle($candidates);
foreach ($candidates as $candidate_resource) {
$blueprint = $blueprints[$candidate_resource->getBlueprintPHID()]->getImplementation();
if ($blueprint->allocateLease($candidate_resource, $lease)) {
$resource = $candidate_resource;
break;
}
}
}
if (!$resource) {
$blueprints = DrydockBlueprintImplementation::getAllBlueprintImplementationsForResource($type);
$this->logToDrydock(pht('Found %d Blueprints', count($blueprints)));
foreach ($blueprints as $key => $candidate_blueprint) {
if (!$candidate_blueprint->isEnabled()) {
unset($blueprints[$key]);
continue;
}
}
$this->logToDrydock(pht('%d Blueprints Enabled', count($blueprints)));
foreach ($blueprints as $key => $candidate_blueprint) {
if (!$candidate_blueprint->canAllocateMoreResources($pool)) {
unset($blueprints[$key]);
continue;
}
}
$this->logToDrydock(pht('%d Blueprints Can Allocate', count($blueprints)));
if (!$blueprints) {
$lease->setStatus(DrydockLeaseStatus::STATUS_BROKEN);
$lease->save();
$this->logToDrydock(pht("There are no resources of type '%s' available, and no " . "blueprints which can allocate new ones.", $type));
return;
}
// TODO: Rank intelligently.
shuffle($blueprints);
$blueprint = head($blueprints);
$resource = $blueprint->allocateResource($lease);
if (!$blueprint->allocateLease($resource, $lease)) {
// TODO: This "should" happen only if we lost a race with another lease,
// which happened to acquire this resource immediately after we
// allocated it. In this case, the right behavior is to retry
// immediately. However, other things like a blueprint allocating a
// resource it can't actually allocate the lease on might be happening
// too, in which case we'd just allocate infinite resources. Probably
// what we should do is test for an active or allocated lease and retry
// if we find one (although it might have already been released by now)
// and fail really hard ("your configuration is a huge broken mess")
// otherwise. But just throw for now since this stuff is all edge-casey.
// Alternatively we could bring resources up in a "BESPOKE" status
// and then switch them to "OPEN" only after the allocating lease gets
// its grubby mitts on the resource. This might make more sense but
// is a bit messy.
throw new Exception(pht('Lost an allocation race?'));
}
}
$blueprint = $resource->getBlueprint();
$blueprint->acquireLease($resource, $lease);
}
protected function reclaimResource(DrydockResource $resource, DrydockLease $lease)
{
$viewer = $this->getViewer();
$command = DrydockCommand::initializeNewCommand($viewer)->setTargetPHID($resource->getPHID())->setAuthorPHID($lease->getPHID())->setCommand(DrydockCommand::COMMAND_RECLAIM)->save();
$resource->scheduleUpdate();
return $this;
}
public function getCommandError(DrydockLease $lease)
{
return $lease->getAttribute('workingcopy.vcs.error');
}
private function validateActivatedLease(DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease)
{
if (!$lease->isActivatedLease()) {
throw new Exception(pht('Blueprint "%s" (of type "%s") is not properly implemented: it ' . 'returned from "%s" without activating a lease.', $blueprint->getBlueprintName(), $blueprint->getClassName(), 'acquireLease()'));
}
}
/**
* @task destroy
*/
private function destroyLease(DrydockLease $lease)
{
$resource = $lease->getResource();
if ($resource) {
$blueprint = $resource->getBlueprint();
$blueprint->destroyLease($resource, $lease);
}
DrydockSlotLock::releaseLocks($lease->getPHID());
$lease->setStatus(DrydockLeaseStatus::STATUS_DESTROYED)->save();
$lease->logEvent(DrydockLeaseDestroyedLogType::LOGCONST);
$lease->awakenTasks();
}
protected function requireActiveLease(DrydockLease $lease)
{
$lease_status = $lease->getStatus();
switch ($lease_status) {
case DrydockLeaseStatus::STATUS_ACQUIRED:
// TODO: Temporary failure.
throw new Exception(pht('Lease still activating.'));
case DrydockLeaseStatus::STATUS_ACTIVE:
return;
default:
// TODO: Permanent failure.
throw new Exception(pht('Lease in bad state.'));
}
}