/**
* @task sync
*/
public function synchronizeWorkingCopyAfterWrite()
{
if (!$this->shouldEnableSynchronization()) {
return;
}
if (!$this->clusterWriteLock) {
throw new Exception(pht('Trying to synchronize after write, but not holding a write ' . 'lock!'));
}
$repository = $this->getRepository();
$repository_phid = $repository->getPHID();
$device = AlmanacKeys::getLiveDevice();
$device_phid = $device->getPHID();
// It is possible that we've lost the global lock while receiving the push.
// For example, the master database may have been restarted between the
// time we acquired the global lock and now, when the push has finished.
// We wrote a durable lock while we were holding the the global lock,
// essentially upgrading our lock. We can still safely release this upgraded
// lock even if we're no longer holding the global lock.
// If we fail to release the lock, the repository will be frozen until
// an operator can figure out what happened, so we try pretty hard to
// reconnect to the database and release the lock.
$now = PhabricatorTime::getNow();
$duration = phutil_units('5 minutes in seconds');
$try_until = $now + $duration;
$did_release = false;
$already_failed = false;
while (PhabricatorTime::getNow() <= $try_until) {
try {
// NOTE: This means we're still bumping the version when pushes fail. We
// could select only un-rejected events instead to bump a little less
// often.
$new_log = id(new PhabricatorRepositoryPushEventQuery())->setViewer(PhabricatorUser::getOmnipotentUser())->withRepositoryPHIDs(array($repository_phid))->setLimit(1)->executeOne();
$old_version = $this->clusterWriteVersion;
if ($new_log) {
$new_version = $new_log->getID();
} else {
$new_version = $old_version;
}
PhabricatorRepositoryWorkingCopyVersion::didWrite($repository_phid, $device_phid, $this->clusterWriteVersion, $new_version, $this->clusterWriteOwner);
$did_release = true;
break;
} catch (AphrontConnectionQueryException $ex) {
$connection_exception = $ex;
} catch (AphrontConnectionLostQueryException $ex) {
$connection_exception = $ex;
}
if (!$already_failed) {
$already_failed = true;
$this->logLine(pht('CRITICAL. Failed to release cluster write lock!'));
$this->logLine(pht('The connection to the master database was lost while receiving ' . 'the write.'));
$this->logLine(pht('This process will spend %s more second(s) attempting to ' . 'recover, then give up.', new PhutilNumber($duration)));
}
sleep(1);
}
if ($did_release) {
if ($already_failed) {
$this->logLine(pht('RECOVERED. Link to master database was restored.'));
}
$this->logLine(pht('Released cluster write lock.'));
} else {
throw new Exception(pht('Failed to reconnect to master database and release held write ' . 'lock ("%s") on device "%s" for repository "%s" after trying ' . 'for %s seconds(s). This repository will be frozen.', $this->clusterWriteOwner, $device->getName(), $this->getDisplayName(), new PhutilNumber($duration)));
}
// We can continue even if we've lost this lock, everything is still
// consistent.
try {
$this->clusterWriteLock->unlock();
} catch (Exception $ex) {
// Ignore.
}
$this->clusterWriteLock = null;
$this->clusterWriteOwner = null;
}
public function execute(PhutilArgumentParser $args)
{
$viewer = $this->getViewer();
$repositories = $this->loadRepositories($args, 'repositories');
if (!$repositories) {
throw new PhutilArgumentUsageException(pht('Specify one or more repositories to thaw.'));
}
$promote = $args->getArg('promote');
$demote = $args->getArg('demote');
if (!$promote && !$demote) {
throw new PhutilArgumentUsageException(pht('You must choose a device to --promote or --demote.'));
}
if ($promote && $demote) {
throw new PhutilArgumentUsageException(pht('Specify either --promote or --demote, but not both.'));
}
$device_name = nonempty($promote, $demote);
$device = id(new AlmanacDeviceQuery())->setViewer($viewer)->withNames(array($device_name))->executeOne();
if (!$device) {
throw new PhutilArgumentUsageException(pht('No device "%s" exists.', $device_name));
}
if ($promote) {
$risk_message = pht('Promoting a device can cause the loss of any repository data which ' . 'only exists on other devices. The version of the repository on the ' . 'promoted device will become authoritative.');
} else {
$risk_message = pht('Demoting a device can cause the loss of any repository data which ' . 'only exists on the demoted device. The version of the repository ' . 'on some other device will become authoritative.');
}
echo tsprintf("**<bg:red> %s </bg>** %s\n", pht('DATA AT RISK'), $risk_message);
$is_force = $args->getArg('force');
$prompt = pht('Accept the possibilty of permanent data loss?');
if (!$is_force && !phutil_console_confirm($prompt)) {
throw new PhutilArgumentUsageException(pht('User aborted the workflow.'));
}
foreach ($repositories as $repository) {
$repository_phid = $repository->getPHID();
$write_lock = PhabricatorRepositoryWorkingCopyVersion::getWriteLock($repository_phid);
echo tsprintf("%s\n", pht('Waiting to acquire write lock for "%s"...', $repository->getDisplayName()));
$write_lock->lock(phutil_units('5 minutes in seconds'));
try {
$service = $repository->loadAlmanacService();
if (!$service) {
throw new PhutilArgumentUsageException(pht('Repository "%s" is not a cluster repository: it is not ' . 'bound to an Almanac service.', $repository->getDisplayName()));
}
$bindings = $service->getActiveBindings();
$bindings = mpull($bindings, null, 'getDevicePHID');
if (empty($bindings[$device->getPHID()])) {
throw new PhutilArgumentUsageException(pht('Repository "%s" has no active binding to device "%s". Only ' . 'actively bound devices can be promoted or demoted.', $repository->getDisplayName(), $device->getName()));
}
$versions = PhabricatorRepositoryWorkingCopyVersion::loadVersions($repository->getPHID());
$versions = mpull($versions, null, 'getDevicePHID');
$versions = array_select_keys($versions, array_keys($bindings));
if ($versions && $promote) {
throw new PhutilArgumentUsageException(pht('Unable to promote "%s" for repository "%s": the leaders for ' . 'this cluster are not ambiguous.', $device->getName(), $repository->getDisplayName()));
}
if ($promote) {
PhabricatorRepositoryWorkingCopyVersion::updateVersion($repository->getPHID(), $device->getPHID(), 0);
echo tsprintf("%s\n", pht('Promoted "%s" to become a leader for "%s".', $device->getName(), $repository->getDisplayName()));
}
if ($demote) {
PhabricatorRepositoryWorkingCopyVersion::demoteDevice($repository->getPHID(), $device->getPHID());
echo tsprintf("%s\n", pht('Demoted "%s" from leadership of repository "%s".', $device->getName(), $repository->getDisplayName()));
}
} catch (Exception $ex) {
$write_lock->unlock();
throw $ex;
}
$write_lock->unlock();
}
return 0;
}
private function buildClusterStatusPanel()
{
$repository = $this->getRepository();
$viewer = $this->getViewer();
$service_phid = $repository->getAlmanacServicePHID();
if ($service_phid) {
$service = id(new AlmanacServiceQuery())->setViewer($viewer)->withServiceTypes(array(AlmanacClusterRepositoryServiceType::SERVICETYPE))->withPHIDs(array($service_phid))->needBindings(true)->executeOne();
if (!$service) {
// TODO: Viewer may not have permission to see the service, or it may
// be invalid? Raise some more useful error here?
throw new Exception(pht('Unable to load cluster service.'));
}
} else {
$service = null;
}
Javelin::initBehavior('phabricator-tooltips');
$rows = array();
if ($service) {
$bindings = $service->getBindings();
$bindings = mgroup($bindings, 'getDevicePHID');
// This is an unusual read which always comes from the master.
if (PhabricatorEnv::isReadOnly()) {
$versions = array();
} else {
$versions = PhabricatorRepositoryWorkingCopyVersion::loadVersions($repository->getPHID());
}
$versions = mpull($versions, null, 'getDevicePHID');
foreach ($bindings as $binding_group) {
$all_disabled = true;
foreach ($binding_group as $binding) {
if (!$binding->getIsDisabled()) {
$all_disabled = false;
break;
}
}
$any_binding = head($binding_group);
if ($all_disabled) {
$binding_icon = 'fa-times grey';
$binding_tip = pht('Disabled');
} else {
$binding_icon = 'fa-folder-open green';
$binding_tip = pht('Active');
}
$binding_icon = id(new PHUIIconView())->setIcon($binding_icon)->addSigil('has-tooltip')->setMetadata(array('tip' => $binding_tip));
$device = $any_binding->getDevice();
$version = idx($versions, $device->getPHID());
if ($version) {
$version_number = $version->getRepositoryVersion();
$href = null;
if ($repository->isHosted()) {
$href = "/diffusion/pushlog/view/{$version_number}/";
} else {
$commit = id(new DiffusionCommitQuery())->setViewer($viewer)->withIDs(array($version_number))->executeOne();
if ($commit) {
$href = $commit->getURI();
}
}
if ($href) {
$version_number = phutil_tag('a', array('href' => $href), $version_number);
}
} else {
$version_number = '-';
}
if ($version && $version->getIsWriting()) {
$is_writing = id(new PHUIIconView())->setIcon('fa-pencil green');
} else {
$is_writing = id(new PHUIIconView())->setIcon('fa-pencil grey');
}
$write_properties = null;
if ($version) {
$write_properties = $version->getWriteProperties();
if ($write_properties) {
try {
$write_properties = phutil_json_decode($write_properties);
} catch (Exception $ex) {
$write_properties = null;
}
}
}
if ($write_properties) {
$writer_phid = idx($write_properties, 'userPHID');
$last_writer = $viewer->renderHandle($writer_phid);
$writer_epoch = idx($write_properties, 'epoch');
$writer_epoch = phabricator_datetime($writer_epoch, $viewer);
} else {
$last_writer = null;
$writer_epoch = null;
}
$rows[] = array($binding_icon, phutil_tag('a', array('href' => $device->getURI()), $device->getName()), $version_number, $is_writing, $last_writer, $writer_epoch);
}
}
$table = id(new AphrontTableView($rows))->setNoDataString(pht('This is not a cluster repository.'))->setHeaders(array(null, pht('Device'), pht('Version'), pht('Writing'), pht('Last Writer'), pht('Last Write At')))->setColumnClasses(array(null, null, null, 'right wide', null, 'date'));
$doc_href = PhabricatorEnv::getDoclink('Cluster: Repositories');
$header = id(new PHUIHeaderView())->setHeader(pht('Cluster Status'))->addActionLink(id(new PHUIButtonView())->setIcon('fa-book')->setHref($doc_href)->setTag('a')->setText(pht('Documentation')));
return id(new PHUIObjectBoxView())->setHeader($header)->setBackground(PHUIObjectBoxView::BLUE_PROPERTY)->setTable($table);
}
