protected function executeRepositoryOperations()
 {
     $repository = $this->getRepository();
     $viewer = $this->getUser();
     $device = AlmanacKeys::getLiveDevice();
     $skip_sync = $this->shouldSkipReadSynchronization();
     if ($this->shouldProxy()) {
         $command = $this->getProxyCommand();
         if ($device) {
             $this->writeClusterEngineLogMessage(pht("# Fetch received by \"%s\", forwarding to cluster host.\n", $device->getName()));
         }
     } else {
         $command = csprintf('git-upload-pack -- %s', $repository->getLocalPath());
         if (!$skip_sync) {
             $cluster_engine = id(new DiffusionRepositoryClusterEngine())->setViewer($viewer)->setRepository($repository)->setLog($this)->synchronizeWorkingCopyBeforeRead();
             if ($device) {
                 $this->writeClusterEngineLogMessage(pht("# Cleared to fetch on cluster host \"%s\".\n", $device->getName()));
             }
         }
     }
     $command = PhabricatorDaemon::sudoCommandAsDaemonUser($command);
     $future = id(new ExecFuture('%C', $command))->setEnv($this->getEnvironment());
     $err = $this->newPassthruCommand()->setIOChannel($this->getIOChannel())->setCommandChannelFromExecFuture($future)->execute();
     if (!$err) {
         $this->waitForGitClient();
     }
     return $err;
 }
 protected function getCurrentDeviceName()
 {
     $device = AlmanacKeys::getLiveDevice();
     if ($device) {
         return $device->getName();
     }
     return php_uname('n');
 }
 protected function newRepositoryLock(PhabricatorRepository $repository, $lock_key, $lock_device_only)
 {
     $lock_parts = array();
     $lock_parts[] = $lock_key;
     $lock_parts[] = $repository->getID();
     if ($lock_device_only) {
         $device = AlmanacKeys::getLiveDevice();
         if ($device) {
             $lock_parts[] = $device->getID();
         }
     }
     $lock_name = implode(':', $lock_parts);
     return PhabricatorGlobalLock::newLock($lock_name);
 }
 protected function loadLocalRepositories(PhutilArgumentParser $args, $param)
 {
     $repositories = $this->loadRepositories($args, $param);
     if (!$repositories) {
         return $repositories;
     }
     $device = AlmanacKeys::getLiveDevice();
     $viewer = $this->getViewer();
     $filter = id(new DiffusionLocalRepositoryFilter())->setViewer($viewer)->setDevice($device)->setRepositories($repositories);
     $repositories = $filter->execute();
     foreach ($filter->getRejectionReasons() as $reason) {
         throw new PhutilArgumentUsageException($reason);
     }
     return $repositories;
 }
 protected function executeRepositoryOperations()
 {
     $repository = $this->getRepository();
     $viewer = $this->getUser();
     $device = AlmanacKeys::getLiveDevice();
     $skip_sync = $this->shouldSkipReadSynchronization();
     $is_proxy = $this->shouldProxy();
     if ($is_proxy) {
         $command = $this->getProxyCommand();
         if ($device) {
             $this->writeClusterEngineLogMessage(pht("# Fetch received by \"%s\", forwarding to cluster host.\n", $device->getName()));
         }
     } else {
         $command = csprintf('git-upload-pack -- %s', $repository->getLocalPath());
         if (!$skip_sync) {
             $cluster_engine = id(new DiffusionRepositoryClusterEngine())->setViewer($viewer)->setRepository($repository)->setLog($this)->synchronizeWorkingCopyBeforeRead();
             if ($device) {
                 $this->writeClusterEngineLogMessage(pht("# Cleared to fetch on cluster host \"%s\".\n", $device->getName()));
             }
         }
     }
     $command = PhabricatorDaemon::sudoCommandAsDaemonUser($command);
     $pull_event = $this->newPullEvent();
     $future = id(new ExecFuture('%C', $command))->setEnv($this->getEnvironment());
     $err = $this->newPassthruCommand()->setIOChannel($this->getIOChannel())->setCommandChannelFromExecFuture($future)->execute();
     if ($err) {
         $pull_event->setResultType('error')->setResultCode($err);
     } else {
         $pull_event->setResultType('pull')->setResultCode(0);
     }
     // TODO: Currently, when proxying, we do not write a log on the proxy.
     // Perhaps we should write a "proxy log". This is not very useful for
     // statistics or auditing, but could be useful for diagnostics. Marking
     // the proxy logs as proxied (and recording devicePHID on all logs) would
     // make differentiating between these use cases easier.
     if (!$is_proxy) {
         $pull_event->save();
     }
     if (!$err) {
         $this->waitForGitClient();
     }
     return $err;
 }
 protected function executeRepositoryOperations()
 {
     $repository = $this->getRepository();
     $viewer = $this->getUser();
     $device = AlmanacKeys::getLiveDevice();
     // This is a write, and must have write access.
     $this->requireWriteAccess();
     $cluster_engine = id(new DiffusionRepositoryClusterEngine())->setViewer($viewer)->setRepository($repository)->setLog($this);
     if ($this->shouldProxy()) {
         $command = $this->getProxyCommand();
         $did_synchronize = false;
         if ($device) {
             $this->writeClusterEngineLogMessage(pht("# Push received by \"%s\", forwarding to cluster host.\n", $device->getName()));
         }
     } else {
         $command = csprintf('git-receive-pack %s', $repository->getLocalPath());
         $did_synchronize = true;
         $cluster_engine->synchronizeWorkingCopyBeforeWrite();
         if ($device) {
             $this->writeClusterEngineLogMessage(pht("# Ready to receive on cluster host \"%s\".\n", $device->getName()));
         }
     }
     $caught = null;
     try {
         $err = $this->executeRepositoryCommand($command);
     } catch (Exception $ex) {
         $caught = $ex;
     }
     // We've committed the write (or rejected it), so we can release the lock
     // without waiting for the client to receive the acknowledgement.
     if ($did_synchronize) {
         $cluster_engine->synchronizeWorkingCopyAfterWrite();
     }
     if ($caught) {
         throw $caught;
     }
     if (!$err) {
         $repository->writeStatusMessage(PhabricatorRepositoryStatusMessage::TYPE_NEEDS_UPDATE, PhabricatorRepositoryStatusMessage::CODE_OKAY);
         $this->waitForGitClient();
     }
     return $err;
 }
 private function newPushLog()
 {
     // NOTE: We generate PHIDs up front so the Herald transcripts can pick them
     // up.
     $phid = id(new PhabricatorRepositoryPushLog())->generatePHID();
     $device = AlmanacKeys::getLiveDevice();
     if ($device) {
         $device_phid = $device->getPHID();
     } else {
         $device_phid = null;
     }
     return PhabricatorRepositoryPushLog::initializeNewLog($this->getViewer())->setPHID($phid)->setDevicePHID($device_phid)->setRepositoryPHID($this->getRepository()->getPHID())->attachRepository($this->getRepository())->setEpoch(time());
 }
示例#8
0
    // up any temporary files we wrote. See T10547.
    exit(128 + $signo);
}
$pattern = array();
$arguments = array();
$pattern[] = 'ssh';
$pattern[] = '-o';
$pattern[] = 'StrictHostKeyChecking=no';
// This prevents "known host" failures, and covers for issues where HOME is set
// to something unusual.
$pattern[] = '-o';
$pattern[] = 'UserKnownHostsFile=/dev/null';
$as_device = getenv('PHABRICATOR_AS_DEVICE');
$credential_phid = getenv('PHABRICATOR_CREDENTIAL');
if ($as_device) {
    $device = AlmanacKeys::getLiveDevice();
    if (!$device) {
        throw new Exception(pht('Attempting to create an SSH connection that authenticates with ' . 'the current device, but this host is not configured as a cluster ' . 'device.'));
    }
    if ($credential_phid) {
        throw new Exception(pht('Attempting to proxy an SSH connection that authenticates with ' . 'both the current device and a specific credential. These options ' . 'are mutually exclusive.'));
    }
}
if ($credential_phid) {
    $viewer = PhabricatorUser::getOmnipotentUser();
    $key = PassphraseSSHKey::loadFromPHID($credential_phid, $viewer);
    $pattern[] = '-l %P';
    $arguments[] = $key->getUsernameEnvelope();
    $pattern[] = '-i %P';
    $arguments[] = $key->getKeyfileEnvelope();
}
 private function newCommonEnvironment()
 {
     $repository = $this->getRepository();
     $env = array();
     // NOTE: Force the language to "en_US.UTF-8", which overrides locale
     // settings. This makes stuff print in English instead of, e.g., French,
     // so we can parse the output of some commands, error messages, etc.
     $env['LANG'] = 'en_US.UTF-8';
     // Propagate PHABRICATOR_ENV explicitly. For discussion, see T4155.
     $env['PHABRICATOR_ENV'] = PhabricatorEnv::getSelectedEnvironmentName();
     $as_device = $this->getConnectAsDevice();
     $credential_phid = $this->getCredentialPHID();
     if ($as_device) {
         $device = AlmanacKeys::getLiveDevice();
         if (!$device) {
             throw new Exception(pht('Attempting to build a reposiory command (for repository "%s") ' . 'as device, but this host ("%s") is not configured as a cluster ' . 'device.', $repository->getDisplayName(), php_uname('n')));
         }
         if ($credential_phid) {
             throw new Exception(pht('Attempting to build a repository command (for repository "%s"), ' . 'but the CommandEngine is configured to connect as both the ' . 'current cluster device ("%s") and with a specific credential ' . '("%s"). These options are mutually exclusive. Connections must ' . 'authenticate as one or the other, not both.', $repository->getDisplayName(), $device->getName(), $credential_phid));
         }
     }
     if ($this->isAnySSHProtocol()) {
         if ($credential_phid) {
             $env['PHABRICATOR_CREDENTIAL'] = $credential_phid;
         }
         if ($as_device) {
             $env['PHABRICATOR_AS_DEVICE'] = 1;
         }
     }
     return $env;
 }
 private function requireWorkingCopy()
 {
     $repository = $this->getRepository();
     $local_path = $repository->getLocalPath();
     if (!Filesystem::pathExists($local_path)) {
         $device = AlmanacKeys::getLiveDevice();
         throw new Exception(pht('Repository "%s" does not have a working copy on this device ' . 'yet, so it can not be synchronized. Wait for the daemons to ' . 'construct one or run `bin/repository update %s` on this host ' . '("%s") to build it explicitly.', $repository->getDisplayName(), $repository->getMonogram(), $device->getName()));
     }
 }
 /**
  * @task pull
  */
 protected function run()
 {
     $argv = $this->getArgv();
     array_unshift($argv, __CLASS__);
     $args = new PhutilArgumentParser($argv);
     $args->parse(array(array('name' => 'no-discovery', 'help' => pht('Pull only, without discovering commits.')), array('name' => 'not', 'param' => 'repository', 'repeat' => true, 'help' => pht('Do not pull __repository__.')), array('name' => 'repositories', 'wildcard' => true, 'help' => pht('Pull specific __repositories__ instead of all.'))));
     $no_discovery = $args->getArg('no-discovery');
     $include = $args->getArg('repositories');
     $exclude = $args->getArg('not');
     // Each repository has an individual pull frequency; after we pull it,
     // wait that long to pull it again. When we start up, try to pull everything
     // serially.
     $retry_after = array();
     $min_sleep = 15;
     $max_futures = 4;
     $futures = array();
     $queue = array();
     while (!$this->shouldExit()) {
         PhabricatorCaches::destroyRequestCache();
         $device = AlmanacKeys::getLiveDevice();
         $pullable = $this->loadPullableRepositories($include, $exclude, $device);
         // If any repositories have the NEEDS_UPDATE flag set, pull them
         // as soon as possible.
         $need_update_messages = $this->loadRepositoryUpdateMessages(true);
         foreach ($need_update_messages as $message) {
             $repo = idx($pullable, $message->getRepositoryID());
             if (!$repo) {
                 continue;
             }
             $this->log(pht('Got an update message for repository "%s"!', $repo->getMonogram()));
             $retry_after[$message->getRepositoryID()] = time();
         }
         // If any repositories were deleted, remove them from the retry timer map
         // so we don't end up with a retry timer that never gets updated and
         // causes us to sleep for the minimum amount of time.
         $retry_after = array_select_keys($retry_after, array_keys($pullable));
         // Figure out which repositories we need to queue for an update.
         foreach ($pullable as $id => $repository) {
             $monogram = $repository->getMonogram();
             if (isset($futures[$id])) {
                 $this->log(pht('Repository "%s" is currently updating.', $monogram));
                 continue;
             }
             if (isset($queue[$id])) {
                 $this->log(pht('Repository "%s" is already queued.', $monogram));
                 continue;
             }
             $after = idx($retry_after, $id, 0);
             if ($after > time()) {
                 $this->log(pht('Repository "%s" is not due for an update for %s second(s).', $monogram, new PhutilNumber($after - time())));
                 continue;
             }
             if (!$after) {
                 $this->log(pht('Scheduling repository "%s" for an initial update.', $monogram));
             } else {
                 $this->log(pht('Scheduling repository "%s" for an update (%s seconds overdue).', $monogram, new PhutilNumber(time() - $after)));
             }
             $queue[$id] = $after;
         }
         // Process repositories in the order they became candidates for updates.
         asort($queue);
         // Dequeue repositories until we hit maximum parallelism.
         while ($queue && count($futures) < $max_futures) {
             foreach ($queue as $id => $time) {
                 $repository = idx($pullable, $id);
                 if (!$repository) {
                     $this->log(pht('Repository %s is no longer pullable; skipping.', $id));
                     unset($queue[$id]);
                     continue;
                 }
                 $monogram = $repository->getMonogram();
                 $this->log(pht('Starting update for repository "%s".', $monogram));
                 unset($queue[$id]);
                 $futures[$id] = $this->buildUpdateFuture($repository, $no_discovery);
                 break;
             }
         }
         if ($queue) {
             $this->log(pht('Not enough process slots to schedule the other %s ' . 'repository(s) for updates yet.', phutil_count($queue)));
         }
         if ($futures) {
             $iterator = id(new FutureIterator($futures))->setUpdateInterval($min_sleep);
             foreach ($iterator as $id => $future) {
                 $this->stillWorking();
                 if ($future === null) {
                     $this->log(pht('Waiting for updates to complete...'));
                     $this->stillWorking();
                     if ($this->loadRepositoryUpdateMessages()) {
                         $this->log(pht('Interrupted by pending updates!'));
                         break;
                     }
                     continue;
                 }
                 unset($futures[$id]);
                 $retry_after[$id] = $this->resolveUpdateFuture($pullable[$id], $future, $min_sleep);
                 // We have a free slot now, so go try to fill it.
                 break;
             }
             // Jump back into prioritization if we had any futures to deal with.
             continue;
         }
         $this->waitForUpdates($min_sleep, $retry_after);
     }
 }