private static function loadPackagesForPaths(PhabricatorRepository $repository, array $paths, $limit = 0)
{
$package = new PhabricatorOwnersPackage();
$path = new PhabricatorOwnersPath();
$conn = $package->establishConnection('r');
$repository_clause = qsprintf($conn, 'AND p.repositoryPHID = %s', $repository->getPHID());
$limit_clause = '';
if (!empty($limit)) {
$limit_clause = qsprintf($conn, 'LIMIT %d', $limit);
}
$data = queryfx_all($conn, 'SELECT pkg.id FROM %T pkg JOIN %T p ON p.packageID = pkg.id
WHERE p.path IN (%Ls) %Q ORDER BY LENGTH(p.path) DESC %Q', $package->getTableName(), $path->getTableName(), $paths, $repository_clause, $limit_clause);
$ids = ipull($data, 'id');
if (empty($ids)) {
return array();
}
$order = array();
foreach ($ids as $id) {
if (empty($order[$id])) {
$order[$id] = true;
}
}
$packages = $package->loadAllWhere('id in (%Ld)', array_keys($order));
$packages = array_select_keys($packages, array_keys($order));
return $packages;
}
private function updateBranchStates(PhabricatorRepository $repository, array $branches)
{
assert_instances_of($branches, 'DiffusionRepositoryRef');
$all_cursors = id(new PhabricatorRepositoryRefCursorQuery())->setViewer(PhabricatorUser::getOmnipotentUser())->withRepositoryPHIDs(array($repository->getPHID()))->execute();
$state_map = array();
$type_branch = PhabricatorRepositoryRefCursor::TYPE_BRANCH;
foreach ($all_cursors as $cursor) {
if ($cursor->getRefType() !== $type_branch) {
continue;
}
$raw_name = $cursor->getRefNameRaw();
$hash = $cursor->getCommitIdentifier();
$state_map[$raw_name][$hash] = $cursor;
}
foreach ($branches as $branch) {
$cursor = idx($state_map, $branch->getShortName(), array());
$cursor = idx($cursor, $branch->getCommitIdentifier());
if (!$cursor) {
continue;
}
$fields = $branch->getRawFields();
$cursor_state = (bool) $cursor->getIsClosed();
$branch_state = (bool) idx($fields, 'closed');
if ($cursor_state != $branch_state) {
$cursor->setIsClosed((int) $branch_state)->save();
}
}
}
public static function newHTTPAuthorization(PhabricatorRepository $repository, PhabricatorUser $viewer, $operation)
{
$lfs_user = self::HTTP_USERNAME;
$lfs_pass = Filesystem::readRandomCharacters(32);
$lfs_hash = PhabricatorHash::digest($lfs_pass);
$ttl = PhabricatorTime::getNow() + phutil_units('1 day in seconds');
$token = id(new PhabricatorAuthTemporaryToken())->setTokenResource($repository->getPHID())->setTokenType(self::TOKENTYPE)->setTokenCode($lfs_hash)->setUserPHID($viewer->getPHID())->setTemporaryTokenProperty('lfs.operation', $operation)->setTokenExpires($ttl)->save();
$authorization_header = base64_encode($lfs_user . ':' . $lfs_pass);
return 'Basic ' . $authorization_header;
}
protected final function parseCommit(PhabricatorRepository $repository, PhabricatorRepositoryCommit $commit)
{
$viewer = PhabricatorUser::getOmnipotentUser();
$refs_raw = DiffusionQuery::callConduitWithDiffusionRequest($viewer, DiffusionRequest::newFromDictionary(array('repository' => $repository, 'user' => $viewer)), 'diffusion.querycommits', array('repositoryPHID' => $repository->getPHID(), 'phids' => array($commit->getPHID()), 'bypassCache' => true, 'needMessages' => true));
if (empty($refs_raw['data'])) {
throw new Exception(pht('Unable to retrieve details for commit "%s"!', $commit->getPHID()));
}
$ref = DiffusionCommitRef::newFromConduitResult(head($refs_raw['data']));
$this->parseCommitWithRef($repository, $commit, $ref);
}
function commit_symbols(array $symbols, PhabricatorRepository $repository, $no_purge)
{
echo pht('Looking up path IDs...'), "\n";
$path_map = PhabricatorRepositoryCommitChangeParserWorker::lookupOrCreatePaths(ipull($symbols, 'path'));
$symbol = new PhabricatorRepositorySymbol();
$conn_w = $symbol->establishConnection('w');
echo pht('Preparing queries...'), "\n";
$sql = array();
foreach ($symbols as $dict) {
$sql[] = qsprintf($conn_w, '(%s, %s, %s, %s, %s, %d, %d)', $repository->getPHID(), $dict['ctxt'], $dict['name'], $dict['type'], $dict['lang'], $dict['line'], $path_map[$dict['path']]);
}
if (!$no_purge) {
echo pht('Purging old symbols...'), "\n";
queryfx($conn_w, 'DELETE FROM %T WHERE repositoryPHID = %s', $symbol->getTableName(), $repository->getPHID());
}
echo pht('Loading %s symbols...', phutil_count($sql)), "\n";
foreach (array_chunk($sql, 128) as $chunk) {
queryfx($conn_w, 'INSERT INTO %T
(repositoryPHID, symbolContext, symbolName, symbolType,
symbolLanguage, lineNumber, pathID) VALUES %Q', $symbol->getTableName(), implode(', ', $chunk));
}
}
protected final function parseCommit(PhabricatorRepository $repository, PhabricatorRepositoryCommit $commit)
{
if (!$this->shouldSkipImportStep()) {
$viewer = PhabricatorUser::getOmnipotentUser();
$refs_raw = DiffusionQuery::callConduitWithDiffusionRequest($viewer, DiffusionRequest::newFromDictionary(array('repository' => $repository, 'user' => $viewer)), 'diffusion.querycommits', array('repositoryPHID' => $repository->getPHID(), 'phids' => array($commit->getPHID()), 'bypassCache' => true, 'needMessages' => true));
if (empty($refs_raw['data'])) {
throw new Exception(pht('Unable to retrieve details for commit "%s"!', $commit->getPHID()));
}
$ref = DiffusionCommitRef::newFromConduitResult(head($refs_raw['data']));
$this->updateCommitData($ref);
}
if ($this->shouldQueueFollowupTasks()) {
$this->queueTask($this->getFollowupTaskClass(), array('commitID' => $commit->getID()), array('priority' => PhabricatorWorker::PRIORITY_DEFAULT));
}
}
private function getHookContextIdentifier(PhabricatorRepository $repository)
{
$identifier = $repository->getPHID();
$instance = PhabricatorEnv::getEnvConfig('cluster.instance');
if (strlen($instance)) {
$identifier = "{$identifier}:{$instance}";
}
return $identifier;
}
private function markUnreachableCommits(PhabricatorRepository $repository)
{
// For now, this is only supported for Git.
if (!$repository->isGit()) {
return;
}
// Find older versions of refs which we haven't processed yet. We're going
// to make sure their commits are still reachable.
$old_refs = id(new PhabricatorRepositoryOldRef())->loadAllWhere('repositoryPHID = %s', $repository->getPHID());
// We can share a single graph stream across all the checks we need to do.
$stream = new PhabricatorGitGraphStream($repository);
foreach ($old_refs as $old_ref) {
$identifier = $old_ref->getCommitIdentifier();
$this->markUnreachableFrom($repository, $stream, $identifier);
// If nothing threw an exception, we're all done with this ref.
$old_ref->delete();
}
}
private static function loadPackagesForPaths(PhabricatorRepository $repository, array $paths, $limit = 0)
{
$package = new PhabricatorOwnersPackage();
$path = new PhabricatorOwnersPath();
$conn = $package->establishConnection('r');
$repository_clause = qsprintf($conn, 'AND p.repositoryPHID = %s', $repository->getPHID());
// NOTE: The list of $paths may be very large if we're coming from
// the OwnersWorker and processing, e.g., an SVN commit which created a new
// branch. Break it apart so that it will fit within 'max_allowed_packet',
// and then merge results in PHP.
$ids = array();
foreach (array_chunk($paths, 128) as $chunk) {
$rows = queryfx_all($conn, 'SELECT pkg.id id, LENGTH(p.path) len
FROM %T pkg JOIN %T p ON p.packageID = pkg.id
WHERE p.path IN (%Ls) %Q', $package->getTableName(), $path->getTableName(), $chunk, $repository_clause);
foreach ($rows as $row) {
$id = (int) $row['id'];
$len = (int) $row['len'];
if (isset($ids[$id])) {
$ids[$id] = max($len, $ids[$id]);
} else {
$ids[$id] = $len;
}
}
}
if (!$ids) {
return array();
}
arsort($ids);
if ($limit) {
$ids = array_slice($ids, 0, $limit, $preserve_keys = true);
}
$ids = array_keys($ids);
$packages = $package->loadAllWhere('id in (%Ld)', $ids);
$packages = array_select_keys($packages, $ids);
return $packages;
}
private function serveGitLFSUploadRequest(PhabricatorRepository $repository, PhabricatorUser $viewer, $oid)
{
$ref = id(new PhabricatorRepositoryGitLFSRefQuery())->setViewer($viewer)->withRepositoryPHIDs(array($repository->getPHID()))->withObjectHashes(array($oid))->executeOne();
if ($ref) {
return DiffusionGitLFSResponse::newErrorResponse(405, pht('Content for object "%s" is already known to this server. It can ' . 'not be uploaded again.', $oid));
}
// Remove the execution time limit because uploading large files may take
// a while.
set_time_limit(0);
$request_stream = new AphrontRequestStream();
$request_iterator = $request_stream->getIterator();
$hashing_iterator = id(new PhutilHashingIterator($request_iterator))->setAlgorithm('sha256');
$source = id(new PhabricatorIteratorFileUploadSource())->setName('lfs-' . $oid)->setViewPolicy(PhabricatorPolicies::POLICY_NOONE)->setIterator($hashing_iterator);
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
$file = $source->uploadFile();
unset($unguarded);
$hash = $hashing_iterator->getHash();
if ($hash !== $oid) {
return DiffusionGitLFSResponse::newErrorResponse(400, pht('Uploaded data is corrupt or invalid. Expected hash "%s", actual ' . 'hash "%s".', $oid, $hash));
}
$ref = id(new PhabricatorRepositoryGitLFSRef())->setRepositoryPHID($repository->getPHID())->setObjectHash($hash)->setByteSize($file->getByteSize())->setAuthorPHID($viewer->getPHID())->setFilePHID($file->getPHID());
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
// Attach the file to the repository to give users permission
// to access it.
$file->attachToObject($repository->getPHID());
$ref->save();
unset($unguarded);
// This is just a plain HTTP 200 with no content, which is what `git lfs`
// expects.
return new DiffusionGitLFSResponse();
}
private function buildBasicProperties(PhabricatorRepository $repository, PhabricatorActionListView $actions)
{
$viewer = $this->getRequest()->getUser();
$view = id(new PHUIPropertyListView())->setUser($viewer)->setActionList($actions);
$type = PhabricatorRepositoryType::getNameForRepositoryType($repository->getVersionControlSystem());
$view->addProperty(pht('Type'), $type);
$view->addProperty(pht('Callsign'), $repository->getCallsign());
$clone_name = $repository->getDetail('clone-name');
if ($repository->isHosted()) {
$view->addProperty(pht('Clone/Checkout As'), $clone_name ? $clone_name . '/' : phutil_tag('em', array(), $repository->getCloneName() . '/'));
}
$project_phids = PhabricatorEdgeQuery::loadDestinationPHIDs($repository->getPHID(), PhabricatorProjectObjectHasProjectEdgeType::EDGECONST);
if ($project_phids) {
$this->loadHandles($project_phids);
$project_text = $this->renderHandlesForPHIDs($project_phids);
} else {
$project_text = phutil_tag('em', array(), pht('None'));
}
$view->addProperty(pht('Projects'), $project_text);
$view->addProperty(pht('Status'), $this->buildRepositoryStatus($repository));
$view->addProperty(pht('Update Frequency'), $this->buildRepositoryUpdateInterval($repository));
$description = $repository->getDetail('description');
$view->addSectionHeader(pht('Description'));
if (!strlen($description)) {
$description = phutil_tag('em', array(), pht('No description provided.'));
} else {
$description = PhabricatorMarkupEngine::renderOneObject($repository, 'description', $viewer);
}
$view->addTextContent($description);
return $view;
}
private function rebuildRepository(PhabricatorRepository $repo)
{
$console = PhutilConsole::getConsole();
$console->writeOut("%s\n", pht('Rebuilding "%s"...', $repo->getMonogram()));
$refs = id(new PhabricatorRepositoryRefCursorQuery())->setViewer($this->getViewer())->withRefTypes(array(PhabricatorRepositoryRefCursor::TYPE_BRANCH))->withRepositoryPHIDs(array($repo->getPHID()))->execute();
$graph = array();
foreach ($refs as $ref) {
if (!$repo->shouldTrackBranch($ref->getRefName())) {
continue;
}
$console->writeOut("%s\n", pht('Rebuilding branch "%s"...', $ref->getRefName()));
$commit = $ref->getCommitIdentifier();
if ($repo->isGit()) {
$stream = new PhabricatorGitGraphStream($repo, $commit);
} else {
$stream = new PhabricatorMercurialGraphStream($repo, $commit);
}
$discover = array($commit);
while ($discover) {
$target = array_pop($discover);
if (isset($graph[$target])) {
continue;
}
$graph[$target] = $stream->getParents($target);
foreach ($graph[$target] as $parent) {
$discover[] = $parent;
}
}
}
$console->writeOut("%s\n", pht('Found %s total commit(s); updating...', phutil_count($graph)));
$commit_table = id(new PhabricatorRepositoryCommit());
$commit_table_name = $commit_table->getTableName();
$conn_w = $commit_table->establishConnection('w');
$bar = id(new PhutilConsoleProgressBar())->setTotal(count($graph));
$need = array();
foreach ($graph as $child => $parents) {
foreach ($parents as $parent) {
$need[$parent] = $parent;
}
$need[$child] = $child;
}
$map = array();
foreach (array_chunk($need, 2048) as $chunk) {
$rows = queryfx_all($conn_w, 'SELECT id, commitIdentifier FROM %T
WHERE commitIdentifier IN (%Ls) AND repositoryID = %d', $commit_table_name, $chunk, $repo->getID());
foreach ($rows as $row) {
$map[$row['commitIdentifier']] = $row['id'];
}
}
$insert_sql = array();
$delete_sql = array();
foreach ($graph as $child => $parents) {
$names = $parents;
$names[] = $child;
foreach ($names as $name) {
if (empty($map[$name])) {
throw new Exception(pht('Unknown commit "%s"!', $name));
}
}
if (!$parents) {
// Write an explicit 0 to indicate "no parents" instead of "no data".
$insert_sql[] = qsprintf($conn_w, '(%d, 0)', $map[$child]);
} else {
foreach ($parents as $parent) {
$insert_sql[] = qsprintf($conn_w, '(%d, %d)', $map[$child], $map[$parent]);
}
}
$delete_sql[] = $map[$child];
$bar->update(1);
}
$commit_table->openTransaction();
foreach (PhabricatorLiskDAO::chunkSQL($delete_sql) as $chunk) {
queryfx($conn_w, 'DELETE FROM %T WHERE childCommitID IN (%Q)', PhabricatorRepository::TABLE_PARENTS, $chunk);
}
foreach (PhabricatorLiskDAO::chunkSQL($insert_sql) as $chunk) {
queryfx($conn_w, 'INSERT INTO %T (childCommitID, parentCommitID) VALUES %Q', PhabricatorRepository::TABLE_PARENTS, $chunk);
}
$commit_table->saveTransaction();
$bar->done();
}
private function newRefQuery(PhabricatorRepository $repository)
{
$viewer = $this->getViewer();
return id(new PhabricatorRepositoryRefCursorQuery())->setViewer($viewer)->withRepositoryPHIDs(array($repository->getPHID()))->withRefTypes($this->getTargetableRefTypes());
}
private function buildAutomationProperties(PhabricatorRepository $repository, PhabricatorActionListView $actions)
{
$viewer = $this->getViewer();
$view = id(new PHUIPropertyListView())->setUser($viewer)->setActionList($actions);
$blueprint_phids = $repository->getAutomationBlueprintPHIDs();
if (!$blueprint_phids) {
$blueprint_view = phutil_tag('em', array(), pht('Not Configured'));
} else {
$blueprint_view = id(new DrydockObjectAuthorizationView())->setUser($viewer)->setObjectPHID($repository->getPHID())->setBlueprintPHIDs($blueprint_phids);
}
$view->addProperty(pht('Automation'), $blueprint_view);
return $view;
}
private function getGitLFSRef(PhabricatorRepository $repository, $data)
{
if (!$repository->canUseGitLFS()) {
return null;
}
$lfs_pattern = '(^version https://git-lfs\\.github\\.com/spec/v1[\\r\\n])';
if (!preg_match($lfs_pattern, $data)) {
return null;
}
$matches = null;
if (!preg_match('(^oid sha256:(.*)$)m', $data, $matches)) {
return null;
}
$hash = $matches[1];
$hash = trim($hash);
return id(new PhabricatorRepositoryGitLFSRefQuery())->setViewer($this->getViewer())->withRepositoryPHIDs(array($repository->getPHID()))->withObjectHashes(array($hash))->executeOne();
}
private function buildSymbolIndexes(PhabricatorRepository $repository, array $visible_changesets)
{
assert_instances_of($visible_changesets, 'DifferentialChangeset');
$engine = PhabricatorSyntaxHighlighter::newEngine();
$langs = $repository->getSymbolLanguages();
$langs = nonempty($langs, array());
$sources = $repository->getSymbolSources();
$sources = nonempty($sources, array());
$symbol_indexes = array();
if ($langs && $sources) {
$have_symbols = id(new DiffusionSymbolQuery())->existsSymbolsInRepository($repository->getPHID());
if (!$have_symbols) {
return $symbol_indexes;
}
}
$repository_phids = array_merge(array($repository->getPHID()), $sources);
$indexed_langs = array_fill_keys($langs, true);
foreach ($visible_changesets as $key => $changeset) {
$lang = $engine->getLanguageFromFilename($changeset->getFilename());
if (empty($indexed_langs) || isset($indexed_langs[$lang])) {
$symbol_indexes[$key] = array('lang' => $lang, 'repositories' => $repository_phids);
}
}
return $symbol_indexes;
}
protected function buildDictForRepository(PhabricatorRepository $repository)
{
return array('name' => $repository->getName(), 'phid' => $repository->getPHID(), 'callsign' => $repository->getCallsign(), 'vcs' => $repository->getVersionControlSystem(), 'uri' => PhabricatorEnv::getProductionURI($repository->getURI()), 'remoteURI' => (string) $repository->getPublicRemoteURI(), 'tracking' => $repository->getDetail('tracking-enabled'), 'description' => $repository->getDetail('description'));
}
private static function loadPackagesForPaths(PhabricatorRepository $repository, array $paths, $limit = 0)
{
$fragments = array();
foreach ($paths as $path) {
foreach (self::splitPath($path) as $fragment) {
$fragments[$fragment][$path] = true;
}
}
$package = new PhabricatorOwnersPackage();
$path = new PhabricatorOwnersPath();
$conn = $package->establishConnection('r');
$repository_clause = qsprintf($conn, 'AND p.repositoryPHID = %s', $repository->getPHID());
// NOTE: The list of $paths may be very large if we're coming from
// the OwnersWorker and processing, e.g., an SVN commit which created a new
// branch. Break it apart so that it will fit within 'max_allowed_packet',
// and then merge results in PHP.
$rows = array();
foreach (array_chunk(array_keys($fragments), 128) as $chunk) {
$rows[] = queryfx_all($conn, 'SELECT pkg.id, p.excluded, p.path
FROM %T pkg JOIN %T p ON p.packageID = pkg.id
WHERE p.path IN (%Ls) %Q', $package->getTableName(), $path->getTableName(), $chunk, $repository_clause);
}
$rows = array_mergev($rows);
$ids = self::findLongestPathsPerPackage($rows, $fragments);
if (!$ids) {
return array();
}
arsort($ids);
if ($limit) {
$ids = array_slice($ids, 0, $limit, $preserve_keys = true);
}
$ids = array_keys($ids);
$packages = $package->loadAllWhere('id in (%Ld)', $ids);
$packages = array_select_keys($packages, $ids);
return $packages;
}
private function markUnreachableCommits(PhabricatorRepository $repository)
{
// For now, this is only supported for Git.
if (!$repository->isGit()) {
return;
}
// Find older versions of refs which we haven't processed yet. We're going
// to make sure their commits are still reachable.
$old_refs = id(new PhabricatorRepositoryOldRef())->loadAllWhere('repositoryPHID = %s', $repository->getPHID());
// If we don't have any refs to update, bail out before building a graph
// stream. In particular, this improves behavior in empty repositories,
// where `git log` exits with an error.
if (!$old_refs) {
return;
}
// We can share a single graph stream across all the checks we need to do.
$stream = new PhabricatorGitGraphStream($repository);
foreach ($old_refs as $old_ref) {
$identifier = $old_ref->getCommitIdentifier();
$this->markUnreachableFrom($repository, $stream, $identifier);
// If nothing threw an exception, we're all done with this ref.
$old_ref->delete();
}
}
