protected function executeRepositoryOperations()
{
$repository = $this->getRepository();
$viewer = $this->getUser();
$device = AlmanacKeys::getLiveDevice();
$skip_sync = $this->shouldSkipReadSynchronization();
if ($this->shouldProxy()) {
$command = $this->getProxyCommand();
if ($device) {
$this->writeClusterEngineLogMessage(pht("# Fetch received by \"%s\", forwarding to cluster host.\n", $device->getName()));
}
} else {
$command = csprintf('git-upload-pack -- %s', $repository->getLocalPath());
if (!$skip_sync) {
$cluster_engine = id(new DiffusionRepositoryClusterEngine())->setViewer($viewer)->setRepository($repository)->setLog($this)->synchronizeWorkingCopyBeforeRead();
if ($device) {
$this->writeClusterEngineLogMessage(pht("# Cleared to fetch on cluster host \"%s\".\n", $device->getName()));
}
}
}
$command = PhabricatorDaemon::sudoCommandAsDaemonUser($command);
$future = id(new ExecFuture('%C', $command))->setEnv($this->getEnvironment());
$err = $this->newPassthruCommand()->setIOChannel($this->getIOChannel())->setCommandChannelFromExecFuture($future)->execute();
if (!$err) {
$this->waitForGitClient();
}
return $err;
}
protected function getCurrentDeviceName()
{
$device = AlmanacKeys::getLiveDevice();
if ($device) {
return $device->getName();
}
return php_uname('n');
}
protected function newRepositoryLock(PhabricatorRepository $repository, $lock_key, $lock_device_only)
{
$lock_parts = array();
$lock_parts[] = $lock_key;
$lock_parts[] = $repository->getID();
if ($lock_device_only) {
$device = AlmanacKeys::getLiveDevice();
if ($device) {
$lock_parts[] = $device->getID();
}
}
$lock_name = implode(':', $lock_parts);
return PhabricatorGlobalLock::newLock($lock_name);
}
protected function loadLocalRepositories(PhutilArgumentParser $args, $param)
{
$repositories = $this->loadRepositories($args, $param);
if (!$repositories) {
return $repositories;
}
$device = AlmanacKeys::getLiveDevice();
$viewer = $this->getViewer();
$filter = id(new DiffusionLocalRepositoryFilter())->setViewer($viewer)->setDevice($device)->setRepositories($repositories);
$repositories = $filter->execute();
foreach ($filter->getRejectionReasons() as $reason) {
throw new PhutilArgumentUsageException($reason);
}
return $repositories;
}
protected function executeRepositoryOperations()
{
$repository = $this->getRepository();
$viewer = $this->getUser();
$device = AlmanacKeys::getLiveDevice();
$skip_sync = $this->shouldSkipReadSynchronization();
$is_proxy = $this->shouldProxy();
if ($is_proxy) {
$command = $this->getProxyCommand();
if ($device) {
$this->writeClusterEngineLogMessage(pht("# Fetch received by \"%s\", forwarding to cluster host.\n", $device->getName()));
}
} else {
$command = csprintf('git-upload-pack -- %s', $repository->getLocalPath());
if (!$skip_sync) {
$cluster_engine = id(new DiffusionRepositoryClusterEngine())->setViewer($viewer)->setRepository($repository)->setLog($this)->synchronizeWorkingCopyBeforeRead();
if ($device) {
$this->writeClusterEngineLogMessage(pht("# Cleared to fetch on cluster host \"%s\".\n", $device->getName()));
}
}
}
$command = PhabricatorDaemon::sudoCommandAsDaemonUser($command);
$pull_event = $this->newPullEvent();
$future = id(new ExecFuture('%C', $command))->setEnv($this->getEnvironment());
$err = $this->newPassthruCommand()->setIOChannel($this->getIOChannel())->setCommandChannelFromExecFuture($future)->execute();
if ($err) {
$pull_event->setResultType('error')->setResultCode($err);
} else {
$pull_event->setResultType('pull')->setResultCode(0);
}
// TODO: Currently, when proxying, we do not write a log on the proxy.
// Perhaps we should write a "proxy log". This is not very useful for
// statistics or auditing, but could be useful for diagnostics. Marking
// the proxy logs as proxied (and recording devicePHID on all logs) would
// make differentiating between these use cases easier.
if (!$is_proxy) {
$pull_event->save();
}
if (!$err) {
$this->waitForGitClient();
}
return $err;
}
protected function executeRepositoryOperations()
{
$repository = $this->getRepository();
$viewer = $this->getUser();
$device = AlmanacKeys::getLiveDevice();
// This is a write, and must have write access.
$this->requireWriteAccess();
$cluster_engine = id(new DiffusionRepositoryClusterEngine())->setViewer($viewer)->setRepository($repository)->setLog($this);
if ($this->shouldProxy()) {
$command = $this->getProxyCommand();
$did_synchronize = false;
if ($device) {
$this->writeClusterEngineLogMessage(pht("# Push received by \"%s\", forwarding to cluster host.\n", $device->getName()));
}
} else {
$command = csprintf('git-receive-pack %s', $repository->getLocalPath());
$did_synchronize = true;
$cluster_engine->synchronizeWorkingCopyBeforeWrite();
if ($device) {
$this->writeClusterEngineLogMessage(pht("# Ready to receive on cluster host \"%s\".\n", $device->getName()));
}
}
$caught = null;
try {
$err = $this->executeRepositoryCommand($command);
} catch (Exception $ex) {
$caught = $ex;
}
// We've committed the write (or rejected it), so we can release the lock
// without waiting for the client to receive the acknowledgement.
if ($did_synchronize) {
$cluster_engine->synchronizeWorkingCopyAfterWrite();
}
if ($caught) {
throw $caught;
}
if (!$err) {
$repository->writeStatusMessage(PhabricatorRepositoryStatusMessage::TYPE_NEEDS_UPDATE, PhabricatorRepositoryStatusMessage::CODE_OKAY);
$this->waitForGitClient();
}
return $err;
}
private function newPushLog()
{
// NOTE: We generate PHIDs up front so the Herald transcripts can pick them
// up.
$phid = id(new PhabricatorRepositoryPushLog())->generatePHID();
$device = AlmanacKeys::getLiveDevice();
if ($device) {
$device_phid = $device->getPHID();
} else {
$device_phid = null;
}
return PhabricatorRepositoryPushLog::initializeNewLog($this->getViewer())->setPHID($phid)->setDevicePHID($device_phid)->setRepositoryPHID($this->getRepository()->getPHID())->attachRepository($this->getRepository())->setEpoch(time());
}
// up any temporary files we wrote. See T10547.
exit(128 + $signo);
}
$pattern = array();
$arguments = array();
$pattern[] = 'ssh';
$pattern[] = '-o';
$pattern[] = 'StrictHostKeyChecking=no';
// This prevents "known host" failures, and covers for issues where HOME is set
// to something unusual.
$pattern[] = '-o';
$pattern[] = 'UserKnownHostsFile=/dev/null';
$as_device = getenv('PHABRICATOR_AS_DEVICE');
$credential_phid = getenv('PHABRICATOR_CREDENTIAL');
if ($as_device) {
$device = AlmanacKeys::getLiveDevice();
if (!$device) {
throw new Exception(pht('Attempting to create an SSH connection that authenticates with ' . 'the current device, but this host is not configured as a cluster ' . 'device.'));
}
if ($credential_phid) {
throw new Exception(pht('Attempting to proxy an SSH connection that authenticates with ' . 'both the current device and a specific credential. These options ' . 'are mutually exclusive.'));
}
}
if ($credential_phid) {
$viewer = PhabricatorUser::getOmnipotentUser();
$key = PassphraseSSHKey::loadFromPHID($credential_phid, $viewer);
$pattern[] = '-l %P';
$arguments[] = $key->getUsernameEnvelope();
$pattern[] = '-i %P';
$arguments[] = $key->getKeyfileEnvelope();
}
private function newCommonEnvironment()
{
$repository = $this->getRepository();
$env = array();
// NOTE: Force the language to "en_US.UTF-8", which overrides locale
// settings. This makes stuff print in English instead of, e.g., French,
// so we can parse the output of some commands, error messages, etc.
$env['LANG'] = 'en_US.UTF-8';
// Propagate PHABRICATOR_ENV explicitly. For discussion, see T4155.
$env['PHABRICATOR_ENV'] = PhabricatorEnv::getSelectedEnvironmentName();
$as_device = $this->getConnectAsDevice();
$credential_phid = $this->getCredentialPHID();
if ($as_device) {
$device = AlmanacKeys::getLiveDevice();
if (!$device) {
throw new Exception(pht('Attempting to build a reposiory command (for repository "%s") ' . 'as device, but this host ("%s") is not configured as a cluster ' . 'device.', $repository->getDisplayName(), php_uname('n')));
}
if ($credential_phid) {
throw new Exception(pht('Attempting to build a repository command (for repository "%s"), ' . 'but the CommandEngine is configured to connect as both the ' . 'current cluster device ("%s") and with a specific credential ' . '("%s"). These options are mutually exclusive. Connections must ' . 'authenticate as one or the other, not both.', $repository->getDisplayName(), $device->getName(), $credential_phid));
}
}
if ($this->isAnySSHProtocol()) {
if ($credential_phid) {
$env['PHABRICATOR_CREDENTIAL'] = $credential_phid;
}
if ($as_device) {
$env['PHABRICATOR_AS_DEVICE'] = 1;
}
}
return $env;
}
private function requireWorkingCopy()
{
$repository = $this->getRepository();
$local_path = $repository->getLocalPath();
if (!Filesystem::pathExists($local_path)) {
$device = AlmanacKeys::getLiveDevice();
throw new Exception(pht('Repository "%s" does not have a working copy on this device ' . 'yet, so it can not be synchronized. Wait for the daemons to ' . 'construct one or run `bin/repository update %s` on this host ' . '("%s") to build it explicitly.', $repository->getDisplayName(), $repository->getMonogram(), $device->getName()));
}
}
/**
* @task pull
*/
protected function run()
{
$argv = $this->getArgv();
array_unshift($argv, __CLASS__);
$args = new PhutilArgumentParser($argv);
$args->parse(array(array('name' => 'no-discovery', 'help' => pht('Pull only, without discovering commits.')), array('name' => 'not', 'param' => 'repository', 'repeat' => true, 'help' => pht('Do not pull __repository__.')), array('name' => 'repositories', 'wildcard' => true, 'help' => pht('Pull specific __repositories__ instead of all.'))));
$no_discovery = $args->getArg('no-discovery');
$include = $args->getArg('repositories');
$exclude = $args->getArg('not');
// Each repository has an individual pull frequency; after we pull it,
// wait that long to pull it again. When we start up, try to pull everything
// serially.
$retry_after = array();
$min_sleep = 15;
$max_futures = 4;
$futures = array();
$queue = array();
while (!$this->shouldExit()) {
PhabricatorCaches::destroyRequestCache();
$device = AlmanacKeys::getLiveDevice();
$pullable = $this->loadPullableRepositories($include, $exclude, $device);
// If any repositories have the NEEDS_UPDATE flag set, pull them
// as soon as possible.
$need_update_messages = $this->loadRepositoryUpdateMessages(true);
foreach ($need_update_messages as $message) {
$repo = idx($pullable, $message->getRepositoryID());
if (!$repo) {
continue;
}
$this->log(pht('Got an update message for repository "%s"!', $repo->getMonogram()));
$retry_after[$message->getRepositoryID()] = time();
}
// If any repositories were deleted, remove them from the retry timer map
// so we don't end up with a retry timer that never gets updated and
// causes us to sleep for the minimum amount of time.
$retry_after = array_select_keys($retry_after, array_keys($pullable));
// Figure out which repositories we need to queue for an update.
foreach ($pullable as $id => $repository) {
$monogram = $repository->getMonogram();
if (isset($futures[$id])) {
$this->log(pht('Repository "%s" is currently updating.', $monogram));
continue;
}
if (isset($queue[$id])) {
$this->log(pht('Repository "%s" is already queued.', $monogram));
continue;
}
$after = idx($retry_after, $id, 0);
if ($after > time()) {
$this->log(pht('Repository "%s" is not due for an update for %s second(s).', $monogram, new PhutilNumber($after - time())));
continue;
}
if (!$after) {
$this->log(pht('Scheduling repository "%s" for an initial update.', $monogram));
} else {
$this->log(pht('Scheduling repository "%s" for an update (%s seconds overdue).', $monogram, new PhutilNumber(time() - $after)));
}
$queue[$id] = $after;
}
// Process repositories in the order they became candidates for updates.
asort($queue);
// Dequeue repositories until we hit maximum parallelism.
while ($queue && count($futures) < $max_futures) {
foreach ($queue as $id => $time) {
$repository = idx($pullable, $id);
if (!$repository) {
$this->log(pht('Repository %s is no longer pullable; skipping.', $id));
unset($queue[$id]);
continue;
}
$monogram = $repository->getMonogram();
$this->log(pht('Starting update for repository "%s".', $monogram));
unset($queue[$id]);
$futures[$id] = $this->buildUpdateFuture($repository, $no_discovery);
break;
}
}
if ($queue) {
$this->log(pht('Not enough process slots to schedule the other %s ' . 'repository(s) for updates yet.', phutil_count($queue)));
}
if ($futures) {
$iterator = id(new FutureIterator($futures))->setUpdateInterval($min_sleep);
foreach ($iterator as $id => $future) {
$this->stillWorking();
if ($future === null) {
$this->log(pht('Waiting for updates to complete...'));
$this->stillWorking();
if ($this->loadRepositoryUpdateMessages()) {
$this->log(pht('Interrupted by pending updates!'));
break;
}
continue;
}
unset($futures[$id]);
$retry_after[$id] = $this->resolveUpdateFuture($pullable[$id], $future, $min_sleep);
// We have a free slot now, so go try to fill it.
break;
}
// Jump back into prioritization if we had any futures to deal with.
continue;
}
$this->waitForUpdates($min_sleep, $retry_after);
}
}