public function execute()
{
$table = new PhabricatorRepositoryCommit();
$conn_r = $table->establishConnection('r');
$join = $this->buildJoinClause($conn_r);
$where = $this->buildWhereClause($conn_r);
$order = $this->buildOrderClause($conn_r);
$limit = $this->buildLimitClause($conn_r);
$data = queryfx_all($conn_r, 'SELECT c.* FROM %T c %Q %Q %Q %Q', $table->getTableName(), $join, $where, $order, $limit);
$commits = $table->loadAllFromArray($data);
if ($this->needCommitData && $commits) {
$data = id(new PhabricatorRepositoryCommitData())->loadAllWhere('commitID in (%Ld)', mpull($commits, 'getID'));
$data = mpull($data, null, 'getCommitID');
foreach ($commits as $commit) {
if (idx($data, $commit->getID())) {
$commit->attachCommitData($data[$commit->getID()]);
} else {
$commit->attachCommitData(new PhabricatorRepositoryCommitData());
}
}
}
if ($this->needAudits && $commits) {
$audits = id(new PhabricatorAuditComment())->loadAllWhere('targetPHID in (%Ls)', mpull($commits, 'getPHID'));
$audits = mgroup($audits, 'getTargetPHID');
foreach ($commits as $commit) {
$commit->attachAudits(idx($audits, $commit->getPHID(), array()));
}
}
return $commits;
}
public function execute(PhutilArgumentParser $args)
{
$repos = $this->loadRepositories($args, 'repos');
if (!$repos) {
throw new PhutilArgumentUsageException(pht('Specify one or more repositories to find importing commits for.'));
}
$repos = mpull($repos, null, 'getID');
$table = new PhabricatorRepositoryCommit();
$conn_r = $table->establishConnection('r');
$rows = queryfx_all($conn_r, 'SELECT repositoryID, commitIdentifier, importStatus FROM %T
WHERE repositoryID IN (%Ld)
AND (importStatus & %d) != %d
AND (importStatus & %d) != %d', $table->getTableName(), array_keys($repos), PhabricatorRepositoryCommit::IMPORTED_ALL, PhabricatorRepositoryCommit::IMPORTED_ALL, PhabricatorRepositoryCommit::IMPORTED_UNREACHABLE, PhabricatorRepositoryCommit::IMPORTED_UNREACHABLE);
$console = PhutilConsole::getConsole();
if ($rows) {
foreach ($rows as $row) {
$repo = $repos[$row['repositoryID']];
$identifier = $row['commitIdentifier'];
$console->writeOut('%s', $repo->formatCommitName($identifier));
if (!$args->getArg('simple')) {
$status = $row['importStatus'];
$need = array();
if (!($status & PhabricatorRepositoryCommit::IMPORTED_MESSAGE)) {
$need[] = pht('Message');
}
if (!($status & PhabricatorRepositoryCommit::IMPORTED_CHANGE)) {
$need[] = pht('Change');
}
if (!($status & PhabricatorRepositoryCommit::IMPORTED_OWNERS)) {
$need[] = pht('Owners');
}
if (!($status & PhabricatorRepositoryCommit::IMPORTED_HERALD)) {
$need[] = pht('Herald');
}
$console->writeOut(' %s', implode(', ', $need));
}
$console->writeOut("\n");
}
} else {
$console->writeErr("%s\n", pht('No importing commits found.'));
}
return 0;
}
public function processRequest()
{
$request = $this->getRequest();
$query = $request->getStr('q');
$repo_id = $request->getInt('repo');
$since = $request->getInt('since');
$limit = $request->getInt('limit');
$now = time();
$data = array();
// Dummy instances used for getting connections, table names, etc.
$pr_commit = new PhabricatorRepositoryCommit();
$pr_commit_data = new PhabricatorRepositoryCommitData();
$conn = $pr_commit->establishConnection('r');
$rows = queryfx_all($conn, 'SELECT
rc.phid as commitPHID,
rc.authorPHID,
rcd.authorName,
SUBSTRING(rcd.commitMessage, 1, 100) AS shortMessage,
rc.commitIdentifier,
rc.epoch
FROM %T rc
INNER JOIN %T rcd ON rcd.commitID = rc.id
WHERE repositoryID = %d
AND rc.epoch >= %d
AND (
rcd.commitMessage LIKE %~
OR
rc.commitIdentifier LIKE %~
)
ORDER BY rc.epoch DESC
LIMIT %d', $pr_commit->getTableName(), $pr_commit_data->getTableName(), $repo_id, $since, $query, $query, $limit);
foreach ($rows as $row) {
$full_commit_id = $row['commitIdentifier'];
$short_commit_id = substr($full_commit_id, 0, 12);
$first_line = $this->getFirstLine($row['shortMessage']);
$data[] = array($full_commit_id, $short_commit_id, $row['authorName'], phutil_format_relative_time($now - $row['epoch']), $first_line);
}
return id(new AphrontAjaxResponse())->setContent($data);
}
<?php
echo pht('Migrating %s to edges...', 'differential.revisionPHID') . "\n";
$commit_table = new PhabricatorRepositoryCommit();
$data_table = new PhabricatorRepositoryCommitData();
$editor = new PhabricatorEdgeEditor();
$commit_table->establishConnection('w');
$edges = 0;
foreach (new LiskMigrationIterator($commit_table) as $commit) {
$data = $commit->loadOneRelative($data_table, 'commitID');
if (!$data) {
continue;
}
$revision_phid = $data->getCommitDetail('differential.revisionPHID');
if (!$revision_phid) {
continue;
}
$commit_drev = DiffusionCommitHasRevisionEdgeType::EDGECONST;
$editor->addEdge($commit->getPHID(), $commit_drev, $revision_phid);
$edges++;
if ($edges % 256 == 0) {
echo '.';
$editor->save();
$editor = new PhabricatorEdgeEditor();
}
}
echo '.';
$editor->save();
echo "\n" . pht('Done.') . "\n";
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
echo "Updating old commit authors...\n";
$table = new PhabricatorRepositoryCommit();
$conn = $table->establishConnection('w');
$data = new PhabricatorRepositoryCommitData();
$commits = queryfx_all($conn, 'SELECT c.id id, c.authorPHID authorPHID, d.commitDetails details
FROM %T c JOIN %T d ON d.commitID = c.id
WHERE c.authorPHID IS NULL', $table->getTableName(), $data->getTableName());
foreach ($commits as $commit) {
$id = $commit['id'];
$details = json_decode($commit['details'], true);
$author_phid = idx($details, 'authorPHID');
if ($author_phid) {
queryfx($conn, 'UPDATE %T SET authorPHID = %s WHERE id = %d', $table->getTableName(), $author_phid, $id);
echo "#{$id}\n";
}
}
echo "Done.\n";
echo "Updating old commit mailKeys...\n";
$table = new PhabricatorRepositoryCommit();
$conn = $table->establishConnection('w');
$commits = queryfx_all($conn, 'SELECT id FROM %T WHERE mailKey = %s', $table->getTableName(), '');
foreach ($commits as $commit) {
$id = $commit['id'];
queryfx($conn, 'UPDATE %T SET mailKey = %s WHERE id = %d', $table->getTableName(), Filesystem::readRandomCharacters(20), $id);
echo "#{$id}\n";
}
echo "Done.\n";
protected function loadPage()
{
$table = new PhabricatorRepositoryCommit();
$conn_r = $table->establishConnection('r');
$data = queryfx_all($conn_r, 'SELECT commit.* FROM %T commit %Q %Q %Q %Q %Q', $table->getTableName(), $this->buildJoinClause($conn_r), $this->buildWhereClause($conn_r), $this->buildGroupClause($conn_r), $this->buildOrderClause($conn_r), $this->buildLimitClause($conn_r));
return $table->loadAllFromArray($data);
}
private function lookupSvnCommits(PhabricatorRepository $repository, array $commits)
{
if (!$commits) {
return array();
}
$commit_table = new PhabricatorRepositoryCommit();
$commit_data = queryfx_all($commit_table->establishConnection('w'), 'SELECT id, commitIdentifier FROM %T
WHERE repositoryID = %d AND commitIdentifier in (%Ls)', $commit_table->getTableName(), $repository->getID(), $commits);
$commit_map = ipull($commit_data, 'id', 'commitIdentifier');
$need = array();
foreach ($commits as $commit) {
if (empty($commit_map[$commit])) {
$need[] = $commit;
}
}
// If we are parsing a Subversion repository and have been configured to
// import only some subdirectory of it, we may find commits which reference
// other foreign commits outside of the directory (for instance, because of
// a move or copy). Rather than trying to execute full parses on them, just
// create stub commits and identify the stubs as foreign commits.
if ($need) {
$subpath = $repository->getDetail('svn-subpath');
if (!$subpath) {
$commits = implode(', ', $need);
throw new Exception("Missing commits ({$need}) in a SVN repository which is not " . "configured for subdirectory-only parsing!");
}
foreach ($need as $foreign_commit) {
$commit = new PhabricatorRepositoryCommit();
$commit->setRepositoryID($repository->getID());
$commit->setCommitIdentifier($foreign_commit);
$commit->setEpoch(0);
// Mark this commit as imported so it doesn't prevent the repository
// from transitioning into the "Imported" state.
$commit->setImportStatus(PhabricatorRepositoryCommit::IMPORTED_ALL);
$commit->save();
$data = new PhabricatorRepositoryCommitData();
$data->setCommitID($commit->getID());
$data->setAuthorName('');
$data->setCommitMessage('');
$data->setCommitDetails(array('foreign-svn-stub' => true, 'svn-subpath' => $subpath));
$data->save();
$commit_map[$foreign_commit] = $commit->getID();
}
}
return $commit_map;
}
if (!phutil_console_confirm('Are you ready to continue?')) {
echo "Cancelled.\n";
exit(1);
}
}
$commits = array();
if ($all_from_repo) {
$repository = id(new PhabricatorRepository())->loadOneWhere('callsign = %s OR phid = %s', $all_from_repo, $all_from_repo);
if (!$repository) {
throw new Exception("Unknown repository {$all_from_repo}!");
}
$constraint = '';
if ($min_timestamp) {
echo "Excluding entries before UNIX timestamp: " . $min_timestamp . "\n";
$table = new PhabricatorRepositoryCommit();
$conn_r = $table->establishConnection('r');
$constraint = qsprintf($conn_r, 'AND epoch >= %d', $min_timestamp);
}
$commits = id(new PhabricatorRepositoryCommit())->loadAllWhere('repositoryID = %d %Q', $repository->getID(), $constraint);
$callsign = $repository->getCallsign();
if (!$commits) {
echo "No commits have been discovered in {$callsign} repository!\n";
exit;
}
} else {
$commits = array();
foreach ($reparse_what as $identifier) {
$matches = null;
if (!preg_match('/r([A-Z]+)([a-z0-9]+)/', $identifier, $matches)) {
throw new Exception("Can't parse commit identifier!");
}
/**
* Mark a list of commits as closeable, and queue workers for those commits
* which don't already have the flag.
*/
private function setCloseFlagOnCommits(array $identifiers)
{
$repository = $this->getRepository();
$commit_table = new PhabricatorRepositoryCommit();
$conn_w = $commit_table->establishConnection('w');
$vcs = $repository->getVersionControlSystem();
switch ($vcs) {
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
$class = 'PhabricatorRepositoryGitCommitMessageParserWorker';
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
$class = 'PhabricatorRepositorySvnCommitMessageParserWorker';
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
$class = 'PhabricatorRepositoryMercurialCommitMessageParserWorker';
break;
default:
throw new Exception("Unknown repository type '{$vcs}'!");
}
$all_commits = queryfx_all($conn_w, 'SELECT id, commitIdentifier, importStatus FROM %T
WHERE repositoryID = %d AND commitIdentifier IN (%Ls)', $commit_table->getTableName(), $repository->getID(), $identifiers);
$closeable_flag = PhabricatorRepositoryCommit::IMPORTED_CLOSEABLE;
$all_commits = ipull($all_commits, null, 'commitIdentifier');
foreach ($identifiers as $identifier) {
$row = idx($all_commits, $identifier);
if (!$row) {
throw new Exception(pht('Commit "%s" has not been discovered yet! Run discovery before ' . 'updating refs.', $identifier));
}
if (!($row['importStatus'] & $closeable_flag)) {
queryfx($conn_w, 'UPDATE %T SET importStatus = (importStatus | %d) WHERE id = %d', $commit_table->getTableName(), $closeable_flag, $row['id']);
$data = array('commitID' => $row['id'], 'only' => true);
PhabricatorWorker::scheduleTask($class, $data);
}
}
}
/**
* Rebuild a cache bucket, amending existing data if available.
*
* @param int Bucket key, from @{method:getBucketKey}.
* @param array Existing bucket data.
* @return array Rebuilt bucket data.
* @task cache
*/
private function rebuildBucket($bucket_key, array $current_data)
{
// First, check if we've already rebuilt this bucket. In some cases (like
// browsing a repository at some commit) it's common to issue many lookups
// against one commit. If that commit has been discovered but not yet
// fully imported, we'll repeatedly attempt to rebuild the bucket. If the
// first rebuild did not work, subsequent rebuilds are very unlikely to
// have any effect. We can just skip the rebuild in these cases.
if (isset($this->rebuiltKeys[$bucket_key])) {
return $current_data;
} else {
$this->rebuiltKeys[$bucket_key] = true;
}
$bucket_min = $bucket_key * $this->getBucketSize();
$bucket_max = $bucket_min + $this->getBucketSize() - 1;
// We need to reload all of the commits in the bucket because there is
// no guarantee that they'll get parsed in order, so we can fill large
// commit IDs before small ones. Later on, we'll ignore the commits we
// already know about.
$table_commit = new PhabricatorRepositoryCommit();
$table_repository = new PhabricatorRepository();
$conn_r = $table_commit->establishConnection('r');
// Find all the Git and Mercurial commits in the block which have completed
// change import. We can't fill the cache accurately for commits which have
// not completed change import, so just pretend we don't know about them.
// In these cases, we will ultimately fall back to VCS queries.
$commit_rows = queryfx_all($conn_r, 'SELECT c.id FROM %T c
JOIN %T r ON c.repositoryID = r.id AND r.versionControlSystem IN (%Ls)
WHERE c.id BETWEEN %d AND %d
AND (c.importStatus & %d) = %d', $table_commit->getTableName(), $table_repository->getTableName(), array(PhabricatorRepositoryType::REPOSITORY_TYPE_GIT, PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL), $bucket_min, $bucket_max, PhabricatorRepositoryCommit::IMPORTED_CHANGE, PhabricatorRepositoryCommit::IMPORTED_CHANGE);
// If we don't have any data, just return the existing data.
if (!$commit_rows) {
return $current_data;
}
// Remove the commits we already have data for. We don't need to rebuild
// these. If there's nothing left, return the existing data.
$commit_ids = ipull($commit_rows, 'id', 'id');
$commit_ids = array_diff_key($commit_ids, $current_data);
if (!$commit_ids) {
return $current_data;
}
// Find all the path changes for the new commits.
$path_changes = queryfx_all($conn_r, 'SELECT commitID, pathID FROM %T
WHERE commitID IN (%Ld)
AND (isDirect = 1 OR changeType = %d)', PhabricatorRepository::TABLE_PATHCHANGE, $commit_ids, DifferentialChangeType::TYPE_CHILD);
$path_changes = igroup($path_changes, 'commitID');
// Find all the parents for the new commits.
$parents = queryfx_all($conn_r, 'SELECT childCommitID, parentCommitID FROM %T
WHERE childCommitID IN (%Ld)
ORDER BY id ASC', PhabricatorRepository::TABLE_PARENTS, $commit_ids);
$parents = igroup($parents, 'childCommitID');
// Build the actual data for the cache.
foreach ($commit_ids as $commit_id) {
$parent_ids = array();
if (!empty($parents[$commit_id])) {
foreach ($parents[$commit_id] as $row) {
$parent_ids[] = (int) $row['parentCommitID'];
}
} else {
// We expect all rows to have parents (commits with no parents get
// an explicit "0" placeholder). If we're in an older repository, the
// parent information might not have been populated yet. Decline to fill
// the cache if we don't have the parent information, since the fill
// will be incorrect.
continue;
}
if (isset($path_changes[$commit_id])) {
$path_ids = $path_changes[$commit_id];
foreach ($path_ids as $key => $path_id) {
$path_ids[$key] = (int) $path_id['pathID'];
}
sort($path_ids);
} else {
$path_ids = array();
}
$value = $parent_ids;
$value[] = null;
foreach ($path_ids as $path_id) {
$value[] = $path_id;
}
$current_data[$commit_id] = $value;
}
return $current_data;
}
echo ' ' . implode("\n ", mpull($bad, 'getCommitIdentifier'));
$ok = phutil_console_confirm("Do you want to delete these commits?");
if (!$ok) {
echo "OK, aborting.\n";
exit(1);
}
echo "Deleting commits";
foreach ($bad as $commit) {
echo ".";
$commit->delete();
}
echo "\nDone.\n";
}
//// Clean Up Links ////////////////////////////////////////////////////////
$table = new PhabricatorRepositoryCommit();
$valid_phids = queryfx_all($table->establishConnection('r'), 'SELECT phid FROM %T', $table->getTableName());
$valid_phids = ipull($valid_phids, null, 'phid');
//////// Differential <-> Diffusion Links //////////////////////////////////
$dx_conn = id(new DifferentialRevision())->establishConnection('w');
$dx_table = DifferentialRevision::TABLE_COMMIT;
$dx_phids = queryfx_all($dx_conn, 'SELECT commitPHID FROM %T', $dx_table);
$bad_phids = array();
foreach ($dx_phids as $dx_phid) {
if (empty($valid_phids[$dx_phid['commitPHID']])) {
$bad_phids[] = $dx_phid['commitPHID'];
}
}
if ($bad_phids) {
echo "Deleting " . count($bad_phids) . " bad Diffusion links...\n";
queryfx($dx_conn, 'DELETE FROM %T WHERE commitPHID IN (%Ls)', $dx_table, $bad_phids);
echo "Done.\n";
private function lookupSvnCommits(PhabricatorRepository $repository, array $commits)
{
if (!$commits) {
return array();
}
$commit_table = new PhabricatorRepositoryCommit();
$commit_data = queryfx_all($commit_table->establishConnection('w'), 'SELECT id, commitIdentifier FROM %T WHERE commitIdentifier in (%Ld)', $commit_table->getTableName(), $commits);
return ipull($commit_data, 'id', 'commitIdentifier');
}
