protected function executeRepositoryOperations()
{
$repository = $this->getRepository();
$viewer = $this->getUser();
$device = AlmanacKeys::getLiveDevice();
$skip_sync = $this->shouldSkipReadSynchronization();
if ($this->shouldProxy()) {
$command = $this->getProxyCommand();
if ($device) {
$this->writeClusterEngineLogMessage(pht("# Fetch received by \"%s\", forwarding to cluster host.\n", $device->getName()));
}
} else {
$command = csprintf('git-upload-pack -- %s', $repository->getLocalPath());
if (!$skip_sync) {
$cluster_engine = id(new DiffusionRepositoryClusterEngine())->setViewer($viewer)->setRepository($repository)->setLog($this)->synchronizeWorkingCopyBeforeRead();
if ($device) {
$this->writeClusterEngineLogMessage(pht("# Cleared to fetch on cluster host \"%s\".\n", $device->getName()));
}
}
}
$command = PhabricatorDaemon::sudoCommandAsDaemonUser($command);
$future = id(new ExecFuture('%C', $command))->setEnv($this->getEnvironment());
$err = $this->newPassthruCommand()->setIOChannel($this->getIOChannel())->setCommandChannelFromExecFuture($future)->execute();
if (!$err) {
$this->waitForGitClient();
}
return $err;
}
protected function getProxyCommand()
{
$uri = new PhutilURI($this->proxyURI);
$username = AlmanacKeys::getClusterSSHUser();
if ($username === null) {
throw new Exception(pht('Unable to determine the username to connect with when trying ' . 'to proxy an SSH request within the Phabricator cluster.'));
}
$port = $uri->getPort();
$host = $uri->getDomain();
$key_path = AlmanacKeys::getKeyPath('device.key');
if (!Filesystem::pathExists($key_path)) {
throw new Exception(pht('Unable to proxy this SSH request within the cluster: this device ' . 'is not registered and has a missing device key (expected to ' . 'find key at "%s").', $key_path));
}
$options = array();
$options[] = '-o';
$options[] = 'StrictHostKeyChecking=no';
$options[] = '-o';
$options[] = 'UserKnownHostsFile=/dev/null';
// This is suppressing "added <address> to the list of known hosts"
// messages, which are confusing and irrelevant when they arise from
// proxied requests. It might also be suppressing lots of useful errors,
// of course. Ideally, we would enforce host keys eventually.
$options[] = '-o';
$options[] = 'LogLevel=quiet';
// NOTE: We prefix the command with "@username", which the far end of the
// connection will parse in order to act as the specified user. This
// behavior is only available to cluster requests signed by a trusted
// device key.
return csprintf('ssh %Ls -l %s -i %s -p %s %s -- %s %Ls', $options, $username, $key_path, $port, $host, '@' . $this->getUser()->getUsername(), $this->getOriginalArguments());
}
protected function newRepositoryLock(PhabricatorRepository $repository, $lock_key, $lock_device_only)
{
$lock_parts = array();
$lock_parts[] = $lock_key;
$lock_parts[] = $repository->getID();
if ($lock_device_only) {
$device = AlmanacKeys::getLiveDevice();
if ($device) {
$lock_parts[] = $device->getID();
}
}
$lock_name = implode(':', $lock_parts);
return PhabricatorGlobalLock::newLock($lock_name);
}
protected function loadLocalRepositories(PhutilArgumentParser $args, $param)
{
$repositories = $this->loadRepositories($args, $param);
if (!$repositories) {
return $repositories;
}
$device = AlmanacKeys::getLiveDevice();
$viewer = $this->getViewer();
$filter = id(new DiffusionLocalRepositoryFilter())->setViewer($viewer)->setDevice($device)->setRepositories($repositories);
$repositories = $filter->execute();
foreach ($filter->getRejectionReasons() as $reason) {
throw new PhutilArgumentUsageException($reason);
}
return $repositories;
}
protected function executeRepositoryOperations()
{
$repository = $this->getRepository();
$viewer = $this->getUser();
$device = AlmanacKeys::getLiveDevice();
$skip_sync = $this->shouldSkipReadSynchronization();
$is_proxy = $this->shouldProxy();
if ($is_proxy) {
$command = $this->getProxyCommand();
if ($device) {
$this->writeClusterEngineLogMessage(pht("# Fetch received by \"%s\", forwarding to cluster host.\n", $device->getName()));
}
} else {
$command = csprintf('git-upload-pack -- %s', $repository->getLocalPath());
if (!$skip_sync) {
$cluster_engine = id(new DiffusionRepositoryClusterEngine())->setViewer($viewer)->setRepository($repository)->setLog($this)->synchronizeWorkingCopyBeforeRead();
if ($device) {
$this->writeClusterEngineLogMessage(pht("# Cleared to fetch on cluster host \"%s\".\n", $device->getName()));
}
}
}
$command = PhabricatorDaemon::sudoCommandAsDaemonUser($command);
$pull_event = $this->newPullEvent();
$future = id(new ExecFuture('%C', $command))->setEnv($this->getEnvironment());
$err = $this->newPassthruCommand()->setIOChannel($this->getIOChannel())->setCommandChannelFromExecFuture($future)->execute();
if ($err) {
$pull_event->setResultType('error')->setResultCode($err);
} else {
$pull_event->setResultType('pull')->setResultCode(0);
}
// TODO: Currently, when proxying, we do not write a log on the proxy.
// Perhaps we should write a "proxy log". This is not very useful for
// statistics or auditing, but could be useful for diagnostics. Marking
// the proxy logs as proxied (and recording devicePHID on all logs) would
// make differentiating between these use cases easier.
if (!$is_proxy) {
$pull_event->save();
}
if (!$err) {
$this->waitForGitClient();
}
return $err;
}
protected function executeRepositoryOperations()
{
$repository = $this->getRepository();
$viewer = $this->getUser();
$device = AlmanacKeys::getLiveDevice();
// This is a write, and must have write access.
$this->requireWriteAccess();
$cluster_engine = id(new DiffusionRepositoryClusterEngine())->setViewer($viewer)->setRepository($repository)->setLog($this);
if ($this->shouldProxy()) {
$command = $this->getProxyCommand();
$did_synchronize = false;
if ($device) {
$this->writeClusterEngineLogMessage(pht("# Push received by \"%s\", forwarding to cluster host.\n", $device->getName()));
}
} else {
$command = csprintf('git-receive-pack %s', $repository->getLocalPath());
$did_synchronize = true;
$cluster_engine->synchronizeWorkingCopyBeforeWrite();
if ($device) {
$this->writeClusterEngineLogMessage(pht("# Ready to receive on cluster host \"%s\".\n", $device->getName()));
}
}
$caught = null;
try {
$err = $this->executeRepositoryCommand($command);
} catch (Exception $ex) {
$caught = $ex;
}
// We've committed the write (or rejected it), so we can release the lock
// without waiting for the client to receive the acknowledgement.
if ($did_synchronize) {
$cluster_engine->synchronizeWorkingCopyAfterWrite();
}
if ($caught) {
throw $caught;
}
if (!$err) {
$repository->writeStatusMessage(PhabricatorRepositoryStatusMessage::TYPE_NEEDS_UPDATE, PhabricatorRepositoryStatusMessage::CODE_OKAY);
$this->waitForGitClient();
}
return $err;
}
/**
* Build a new Conduit client in order to make a service call to this
* repository.
*
* If the repository is hosted locally, this method may return `null`. The
* caller should use `ConduitCall` or other local logic to complete the
* request.
*
* By default, we will return a @{class:ConduitClient} for any repository with
* a service, even if that service is on the current device.
*
* We do this because this configuration does not make very much sense in a
* production context, but is very common in a test/development context
* (where the developer's machine is both the web host and the repository
* service). By proxying in development, we get more consistent behavior
* between development and production, and don't have a major untested
* codepath.
*
* The `$never_proxy` parameter can be used to prevent this local proxying.
* If the flag is passed:
*
* - The method will return `null` (implying a local service call)
* if the repository service is hosted on the current device.
* - The method will throw if it would need to return a client.
*
* This is used to prevent loops in Conduit: the first request will proxy,
* even in development, but the second request will be identified as a
* cluster request and forced not to proxy.
*
* For lower-level service resolution, see @{method:getAlmanacServiceURI}.
*
* @param PhabricatorUser Viewing user.
* @param bool `true` to throw if a client would be returned.
* @return ConduitClient|null Client, or `null` for local repositories.
*/
public function newConduitClient(PhabricatorUser $viewer, $never_proxy = false)
{
$uri = $this->getAlmanacServiceURI($viewer, $never_proxy, array('http', 'https'));
if ($uri === null) {
return null;
}
$domain = id(new PhutilURI(PhabricatorEnv::getURI('/')))->getDomain();
$client = id(new ConduitClient($uri))->setHost($domain);
if ($viewer->isOmnipotent()) {
// If the caller is the omnipotent user (normally, a daemon), we will
// sign the request with this host's asymmetric keypair.
$public_path = AlmanacKeys::getKeyPath('device.pub');
try {
$public_key = Filesystem::readFile($public_path);
} catch (Exception $ex) {
throw new PhutilAggregateException(pht('Unable to read device public key while attempting to make ' . 'authenticated method call within the Phabricator cluster. ' . 'Use `%s` to register keys for this device. Exception: %s', 'bin/almanac register', $ex->getMessage()), array($ex));
}
$private_path = AlmanacKeys::getKeyPath('device.key');
try {
$private_key = Filesystem::readFile($private_path);
$private_key = new PhutilOpaqueEnvelope($private_key);
} catch (Exception $ex) {
throw new PhutilAggregateException(pht('Unable to read device private key while attempting to make ' . 'authenticated method call within the Phabricator cluster. ' . 'Use `%s` to register keys for this device. Exception: %s', 'bin/almanac register', $ex->getMessage()), array($ex));
}
$client->setSigningKeys($public_key, $private_key);
} else {
// If the caller is a normal user, we generate or retrieve a cluster
// API token.
$token = PhabricatorConduitToken::loadClusterTokenForUser($viewer);
if ($token) {
$client->setConduitToken($token->getToken());
}
}
return $client;
}
private function newPushLog()
{
// NOTE: We generate PHIDs up front so the Herald transcripts can pick them
// up.
$phid = id(new PhabricatorRepositoryPushLog())->generatePHID();
$device = AlmanacKeys::getLiveDevice();
if ($device) {
$device_phid = $device->getPHID();
} else {
$device_phid = null;
}
return PhabricatorRepositoryPushLog::initializeNewLog($this->getViewer())->setPHID($phid)->setDevicePHID($device_phid)->setRepositoryPHID($this->getRepository()->getPHID())->attachRepository($this->getRepository())->setEpoch(time());
}
private function getForcedUser()
{
switch ($this->getBuiltinProtocol()) {
case self::BUILTIN_PROTOCOL_SSH:
return AlmanacKeys::getClusterSSHUser();
default:
return null;
}
}
public function execute(PhutilArgumentParser $args)
{
$viewer = $this->getViewer();
$device_name = $args->getArg('device');
if (!strlen($device_name)) {
throw new PhutilArgumentUsageException(pht('Specify a device with --device.'));
}
$device = id(new AlmanacDeviceQuery())->setViewer($viewer)->withNames(array($device_name))->executeOne();
if (!$device) {
throw new PhutilArgumentUsageException(pht('No such device "%s" exists!', $device_name));
}
$identify_as = $args->getArg('identify-as');
$raw_device = $device_name;
if (strlen($identify_as)) {
$raw_device = $identify_as;
}
$identity_device = id(new AlmanacDeviceQuery())->setViewer($viewer)->withNames(array($raw_device))->executeOne();
if (!$identity_device) {
throw new PhutilArgumentUsageException(pht('No such device "%s" exists!', $raw_device));
}
$private_key_path = $args->getArg('private-key');
if (!strlen($private_key_path)) {
throw new PhutilArgumentUsageException(pht('Specify a private key with --private-key.'));
}
if (!Filesystem::pathExists($private_key_path)) {
throw new PhutilArgumentUsageException(pht('No private key exists at path "%s"!', $private_key_path));
}
$raw_private_key = Filesystem::readFile($private_key_path);
$phd_user = PhabricatorEnv::getEnvConfig('phd.user');
if (!$phd_user) {
throw new PhutilArgumentUsageException(pht('Config option "phd.user" is not set. You must set this option ' . 'so the private key can be stored with the correct permissions.'));
}
$tmp = new TempFile();
list($err) = exec_manual('chown %s %s', $phd_user, $tmp);
if ($err) {
throw new PhutilArgumentUsageException(pht('Unable to change ownership of an identity file to daemon user ' . '"%s". Run this command as %s or root.', $phd_user, $phd_user));
}
$stored_public_path = AlmanacKeys::getKeyPath('device.pub');
$stored_private_path = AlmanacKeys::getKeyPath('device.key');
$stored_device_path = AlmanacKeys::getKeyPath('device.id');
if (!$args->getArg('force')) {
if (Filesystem::pathExists($stored_public_path)) {
throw new PhutilArgumentUsageException(pht('This host already has a registered public key ("%s"). ' . 'Remove this key before registering the host, or use ' . '--force to overwrite it.', Filesystem::readablePath($stored_public_path)));
}
if (Filesystem::pathExists($stored_private_path)) {
throw new PhutilArgumentUsageException(pht('This host already has a registered private key ("%s"). ' . 'Remove this key before registering the host, or use ' . '--force to overwrite it.', Filesystem::readablePath($stored_private_path)));
}
}
// NOTE: We're writing the private key here so we can change permissions
// on it without causing weird side effects to the file specified with
// the `--private-key` flag. The file needs to have restrictive permissions
// before `ssh-keygen` will willingly operate on it.
$tmp_private = new TempFile();
Filesystem::changePermissions($tmp_private, 0600);
execx('chown %s %s', $phd_user, $tmp_private);
Filesystem::writeFile($tmp_private, $raw_private_key);
list($raw_public_key) = execx('ssh-keygen -y -f %s', $tmp_private);
$key_object = PhabricatorAuthSSHPublicKey::newFromRawKey($raw_public_key);
$public_key = id(new PhabricatorAuthSSHKeyQuery())->setViewer($this->getViewer())->withKeys(array($key_object))->withIsActive(true)->executeOne();
if (!$public_key) {
throw new PhutilArgumentUsageException(pht('The public key corresponding to the given private key is not ' . 'yet known to Phabricator. Associate the public key with an ' . 'Almanac device in the web interface before registering hosts ' . 'with it.'));
}
if ($public_key->getObjectPHID() !== $device->getPHID()) {
$public_phid = $public_key->getObjectPHID();
$public_handles = $viewer->loadHandles(array($public_phid));
$public_handle = $public_handles[$public_phid];
throw new PhutilArgumentUsageException(pht('The public key corresponding to the given private key is already ' . 'associated with an object ("%s") other than the specified ' . 'device ("%s"). You can not use a single private key to identify ' . 'multiple devices or users.', $public_handle->getFullName(), $device->getName()));
}
if (!$public_key->getIsTrusted()) {
throw new PhutilArgumentUsageException(pht('The public key corresponding to the given private key is ' . 'properly associated with the device, but is not yet trusted. ' . 'Trust this key before registering devices with it.'));
}
echo tsprintf("%s\n", pht('Installing public key...'));
$tmp_public = new TempFile();
Filesystem::changePermissions($tmp_public, 0600);
execx('chown %s %s', $phd_user, $tmp_public);
Filesystem::writeFile($tmp_public, $raw_public_key);
execx('mv -f %s %s', $tmp_public, $stored_public_path);
echo tsprintf("%s\n", pht('Installing private key...'));
execx('mv -f %s %s', $tmp_private, $stored_private_path);
echo tsprintf("%s\n", pht('Installing device %s...', $raw_device));
// The permissions on this file are more open because the webserver also
// needs to read it.
$tmp_device = new TempFile();
Filesystem::changePermissions($tmp_device, 0644);
execx('chown %s %s', $phd_user, $tmp_device);
Filesystem::writeFile($tmp_device, $raw_device);
execx('mv -f %s %s', $tmp_device, $stored_device_path);
echo tsprintf("**<bg:green> %s </bg>** %s\n", pht('HOST REGISTERED'), pht('This host has been registered as "%s" and a trusted keypair ' . 'has been installed.', $raw_device));
}
throw new Exception(pht('Attempting to proxy an SSH connection that authenticates with ' . 'both the current device and a specific credential. These options ' . 'are mutually exclusive.'));
}
}
if ($credential_phid) {
$viewer = PhabricatorUser::getOmnipotentUser();
$key = PassphraseSSHKey::loadFromPHID($credential_phid, $viewer);
$pattern[] = '-l %P';
$arguments[] = $key->getUsernameEnvelope();
$pattern[] = '-i %P';
$arguments[] = $key->getKeyfileEnvelope();
}
if ($as_device) {
$pattern[] = '-l %R';
$arguments[] = AlmanacKeys::getClusterSSHUser();
$pattern[] = '-i %R';
$arguments[] = AlmanacKeys::getKeyPath('device.key');
}
// Subversion passes us a host in the form "domain.com:port", which is not
// valid for normal SSH but which we can parse into a valid "-p" flag.
$passthru_args = $unconsumed_argv;
$host = array_shift($passthru_args);
$parts = explode(':', $host, 2);
$host = $parts[0];
$port = $args->getArg('port');
if (!$port) {
if (count($parts) == 2) {
$port = $parts[1];
}
}
if ($port) {
$pattern[] = '-p %d';
private function newCommonEnvironment()
{
$repository = $this->getRepository();
$env = array();
// NOTE: Force the language to "en_US.UTF-8", which overrides locale
// settings. This makes stuff print in English instead of, e.g., French,
// so we can parse the output of some commands, error messages, etc.
$env['LANG'] = 'en_US.UTF-8';
// Propagate PHABRICATOR_ENV explicitly. For discussion, see T4155.
$env['PHABRICATOR_ENV'] = PhabricatorEnv::getSelectedEnvironmentName();
$as_device = $this->getConnectAsDevice();
$credential_phid = $this->getCredentialPHID();
if ($as_device) {
$device = AlmanacKeys::getLiveDevice();
if (!$device) {
throw new Exception(pht('Attempting to build a reposiory command (for repository "%s") ' . 'as device, but this host ("%s") is not configured as a cluster ' . 'device.', $repository->getDisplayName(), php_uname('n')));
}
if ($credential_phid) {
throw new Exception(pht('Attempting to build a repository command (for repository "%s"), ' . 'but the CommandEngine is configured to connect as both the ' . 'current cluster device ("%s") and with a specific credential ' . '("%s"). These options are mutually exclusive. Connections must ' . 'authenticate as one or the other, not both.', $repository->getDisplayName(), $device->getName(), $credential_phid));
}
}
if ($this->isAnySSHProtocol()) {
if ($credential_phid) {
$env['PHABRICATOR_CREDENTIAL'] = $credential_phid;
}
if ($as_device) {
$env['PHABRICATOR_AS_DEVICE'] = 1;
}
}
return $env;
}
private function requireWorkingCopy()
{
$repository = $this->getRepository();
$local_path = $repository->getLocalPath();
if (!Filesystem::pathExists($local_path)) {
$device = AlmanacKeys::getLiveDevice();
throw new Exception(pht('Repository "%s" does not have a working copy on this device ' . 'yet, so it can not be synchronized. Wait for the daemons to ' . 'construct one or run `bin/repository update %s` on this host ' . '("%s") to build it explicitly.', $repository->getDisplayName(), $repository->getMonogram(), $device->getName()));
}
}
public function execute(PhutilArgumentParser $args)
{
$console = PhutilConsole::getConsole();
$device_name = $args->getArg('device');
if (!strlen($device_name)) {
throw new PhutilArgumentUsageException(pht('Specify a device with --device.'));
}
$device = id(new AlmanacDeviceQuery())->setViewer($this->getViewer())->withNames(array($device_name))->executeOne();
if (!$device) {
throw new PhutilArgumentUsageException(pht('No such device "%s" exists!', $device_name));
}
$private_key_path = $args->getArg('private-key');
if (!strlen($private_key_path)) {
throw new PhutilArgumentUsageException(pht('Specify a private key with --private-key.'));
}
if (!Filesystem::pathExists($private_key_path)) {
throw new PhutilArgumentUsageException(pht('Private key "%s" does not exist!', $private_key_path));
}
$raw_private_key = Filesystem::readFile($private_key_path);
$phd_user = PhabricatorEnv::getEnvConfig('phd.user');
if (!$phd_user) {
throw new PhutilArgumentUsageException(pht('Config option "phd.user" is not set. You must set this option ' . 'so the private key can be stored with the correct permissions.'));
}
$tmp = new TempFile();
list($err) = exec_manual('chown %s %s', $phd_user, $tmp);
if ($err) {
throw new PhutilArgumentUsageException(pht('Unable to change ownership of a file to daemon user "%s". Run ' . 'this command as %s or root.', $phd_user, $phd_user));
}
$stored_public_path = AlmanacKeys::getKeyPath('device.pub');
$stored_private_path = AlmanacKeys::getKeyPath('device.key');
$stored_device_path = AlmanacKeys::getKeyPath('device.id');
if (!$args->getArg('force')) {
if (Filesystem::pathExists($stored_public_path)) {
throw new PhutilArgumentUsageException(pht('This host already has a registered public key ("%s"). ' . 'Remove this key before registering the host, or use ' . '--force to overwrite it.', Filesystem::readablePath($stored_public_path)));
}
if (Filesystem::pathExists($stored_private_path)) {
throw new PhutilArgumentUsageException(pht('This host already has a registered private key ("%s"). ' . 'Remove this key before registering the host, or use ' . '--force to overwrite it.', Filesystem::readablePath($stored_private_path)));
}
}
// NOTE: We're writing the private key here so we can change permissions
// on it without causing weird side effects to the file specified with
// the `--private-key` flag. The file needs to have restrictive permissions
// before `ssh-keygen` will willingly operate on it.
$tmp_private = new TempFile();
Filesystem::changePermissions($tmp_private, 0600);
execx('chown %s %s', $phd_user, $tmp_private);
Filesystem::writeFile($tmp_private, $raw_private_key);
list($raw_public_key) = execx('ssh-keygen -y -f %s', $tmp_private);
$key_object = PhabricatorAuthSSHPublicKey::newFromRawKey($raw_public_key);
$public_key = id(new PhabricatorAuthSSHKeyQuery())->setViewer($this->getViewer())->withKeys(array($key_object))->executeOne();
if ($public_key) {
if ($public_key->getObjectPHID() !== $device->getPHID()) {
throw new PhutilArgumentUsageException(pht('The public key corresponding to the given private key is ' . 'already associated with an object other than the specified ' . 'device. You can not use a single private key to identify ' . 'multiple devices or users.'));
} else {
if (!$public_key->getIsTrusted()) {
throw new PhutilArgumentUsageException(pht('The public key corresponding to the given private key is ' . 'already associated with the device, but is not trusted. ' . 'Registering this key would trust the other entities which ' . 'hold it. Use a unique key, or explicitly enable trust for the ' . 'current key.'));
} else {
if (!$args->getArg('allow-key-reuse')) {
throw new PhutilArgumentUsageException(pht('The public key corresponding to the given private key is ' . 'already associated with the device. If you do not want to ' . 'use a unique key, use --allow-key-reuse to permit ' . 'reassociation.'));
}
}
}
} else {
$public_key = id(new PhabricatorAuthSSHKey())->setObjectPHID($device->getPHID())->attachObject($device)->setName($device->getSSHKeyDefaultName())->setKeyType($key_object->getType())->setKeyBody($key_object->getBody())->setKeyComment(pht('Registered'))->setIsTrusted(1);
}
$console->writeOut("%s\n", pht('Installing public key...'));
$tmp_public = new TempFile();
Filesystem::changePermissions($tmp_public, 0600);
execx('chown %s %s', $phd_user, $tmp_public);
Filesystem::writeFile($tmp_public, $raw_public_key);
execx('mv -f %s %s', $tmp_public, $stored_public_path);
$console->writeOut("%s\n", pht('Installing private key...'));
execx('mv -f %s %s', $tmp_private, $stored_private_path);
$raw_device = $device_name;
$identify_as = $args->getArg('identify-as');
if (strlen($identify_as)) {
$raw_device = $identify_as;
}
$console->writeOut("%s\n", pht('Installing device %s...', $raw_device));
// The permissions on this file are more open because the webserver also
// needs to read it.
$tmp_device = new TempFile();
Filesystem::changePermissions($tmp_device, 0644);
execx('chown %s %s', $phd_user, $tmp_device);
Filesystem::writeFile($tmp_device, $raw_device);
execx('mv -f %s %s', $tmp_device, $stored_device_path);
if (!$public_key->getID()) {
$console->writeOut("%s\n", pht('Registering device key...'));
$public_key->save();
}
$console->writeOut("**<bg:green> %s </bg>** %s\n", pht('HOST REGISTERED'), pht('This host has been registered as "%s" and a trusted keypair ' . 'has been installed.', $raw_device));
}
/**
* @task pull
*/
protected function run()
{
$argv = $this->getArgv();
array_unshift($argv, __CLASS__);
$args = new PhutilArgumentParser($argv);
$args->parse(array(array('name' => 'no-discovery', 'help' => pht('Pull only, without discovering commits.')), array('name' => 'not', 'param' => 'repository', 'repeat' => true, 'help' => pht('Do not pull __repository__.')), array('name' => 'repositories', 'wildcard' => true, 'help' => pht('Pull specific __repositories__ instead of all.'))));
$no_discovery = $args->getArg('no-discovery');
$include = $args->getArg('repositories');
$exclude = $args->getArg('not');
// Each repository has an individual pull frequency; after we pull it,
// wait that long to pull it again. When we start up, try to pull everything
// serially.
$retry_after = array();
$min_sleep = 15;
$max_futures = 4;
$futures = array();
$queue = array();
while (!$this->shouldExit()) {
PhabricatorCaches::destroyRequestCache();
$device = AlmanacKeys::getLiveDevice();
$pullable = $this->loadPullableRepositories($include, $exclude, $device);
// If any repositories have the NEEDS_UPDATE flag set, pull them
// as soon as possible.
$need_update_messages = $this->loadRepositoryUpdateMessages(true);
foreach ($need_update_messages as $message) {
$repo = idx($pullable, $message->getRepositoryID());
if (!$repo) {
continue;
}
$this->log(pht('Got an update message for repository "%s"!', $repo->getMonogram()));
$retry_after[$message->getRepositoryID()] = time();
}
// If any repositories were deleted, remove them from the retry timer map
// so we don't end up with a retry timer that never gets updated and
// causes us to sleep for the minimum amount of time.
$retry_after = array_select_keys($retry_after, array_keys($pullable));
// Figure out which repositories we need to queue for an update.
foreach ($pullable as $id => $repository) {
$monogram = $repository->getMonogram();
if (isset($futures[$id])) {
$this->log(pht('Repository "%s" is currently updating.', $monogram));
continue;
}
if (isset($queue[$id])) {
$this->log(pht('Repository "%s" is already queued.', $monogram));
continue;
}
$after = idx($retry_after, $id, 0);
if ($after > time()) {
$this->log(pht('Repository "%s" is not due for an update for %s second(s).', $monogram, new PhutilNumber($after - time())));
continue;
}
if (!$after) {
$this->log(pht('Scheduling repository "%s" for an initial update.', $monogram));
} else {
$this->log(pht('Scheduling repository "%s" for an update (%s seconds overdue).', $monogram, new PhutilNumber(time() - $after)));
}
$queue[$id] = $after;
}
// Process repositories in the order they became candidates for updates.
asort($queue);
// Dequeue repositories until we hit maximum parallelism.
while ($queue && count($futures) < $max_futures) {
foreach ($queue as $id => $time) {
$repository = idx($pullable, $id);
if (!$repository) {
$this->log(pht('Repository %s is no longer pullable; skipping.', $id));
unset($queue[$id]);
continue;
}
$monogram = $repository->getMonogram();
$this->log(pht('Starting update for repository "%s".', $monogram));
unset($queue[$id]);
$futures[$id] = $this->buildUpdateFuture($repository, $no_discovery);
break;
}
}
if ($queue) {
$this->log(pht('Not enough process slots to schedule the other %s ' . 'repository(s) for updates yet.', phutil_count($queue)));
}
if ($futures) {
$iterator = id(new FutureIterator($futures))->setUpdateInterval($min_sleep);
foreach ($iterator as $id => $future) {
$this->stillWorking();
if ($future === null) {
$this->log(pht('Waiting for updates to complete...'));
$this->stillWorking();
if ($this->loadRepositoryUpdateMessages()) {
$this->log(pht('Interrupted by pending updates!'));
break;
}
continue;
}
unset($futures[$id]);
$retry_after[$id] = $this->resolveUpdateFuture($pullable[$id], $future, $min_sleep);
// We have a free slot now, so go try to fill it.
break;
}
// Jump back into prioritization if we had any futures to deal with.
continue;
}
$this->waitForUpdates($min_sleep, $retry_after);
}
}
