private function uploadChunks($file_phid, $path)
{
$conduit = $this->getConduit();
$f = @fopen($path, 'rb');
if (!$f) {
throw new Exception(pht('Unable to open file "%s"', $path));
}
$this->writeStatus(pht('Beginning chunked upload of large file...'));
$chunks = $conduit->callMethodSynchronous('file.querychunks', array('filePHID' => $file_phid));
$remaining = array();
foreach ($chunks as $chunk) {
if (!$chunk['complete']) {
$remaining[] = $chunk;
}
}
$done = count($chunks) - count($remaining);
if ($done) {
$this->writeStatus(pht('Resuming upload (%d of %d chunks remain).', new PhutilNumber(count($remaining)), new PhutilNumber(count($chunks))));
} else {
$this->writeStatus(pht('Uploading chunks (%d chunks to upload).', new PhutilNumber(count($remaining))));
}
$progress = new PhutilConsoleProgressBar();
$progress->setTotal(count($chunks));
for ($ii = 0; $ii < $done; $ii++) {
$progress->update(1);
}
$progress->draw();
// TODO: We could do these in parallel to improve upload performance.
foreach ($remaining as $chunk) {
$offset = $chunk['byteStart'];
$ok = fseek($f, $offset);
if ($ok !== 0) {
throw new Exception(pht('Failed to %s!', 'fseek()'));
}
$data = fread($f, $chunk['byteEnd'] - $chunk['byteStart']);
if ($data === false) {
throw new Exception(pht('Failed to %s!', 'fread()'));
}
$conduit->callMethodSynchronous('file.uploadchunk', array('filePHID' => $file_phid, 'byteStart' => $offset, 'dataEncoding' => 'base64', 'data' => base64_encode($data)));
$progress->update(1);
}
}
/**
* Upload missing chunks of a large file by calling `file.uploadchunk` over
* Conduit.
*
* @task internal
*/
private function uploadChunks(ArcanistFileDataRef $file, $file_phid)
{
$conduit = $this->conduit;
$chunks = $conduit->callMethodSynchronous('file.querychunks', array('filePHID' => $file_phid));
$remaining = array();
foreach ($chunks as $chunk) {
if (!$chunk['complete']) {
$remaining[] = $chunk;
}
}
$done = count($chunks) - count($remaining);
if ($done) {
$this->writeStatus(pht('Resuming upload (%s of %s chunks remain).', phutil_count($remaining), phutil_count($chunks)));
} else {
$this->writeStatus(pht('Uploading chunks (%s chunks to upload).', phutil_count($remaining)));
}
$progress = new PhutilConsoleProgressBar();
$progress->setTotal(count($chunks));
for ($ii = 0; $ii < $done; $ii++) {
$progress->update(1);
}
$progress->draw();
// TODO: We could do these in parallel to improve upload performance.
foreach ($remaining as $chunk) {
$data = $file->readBytes($chunk['byteStart'], $chunk['byteEnd']);
$conduit->callMethodSynchronous('file.uploadchunk', array('filePHID' => $file_phid, 'byteStart' => $chunk['byteStart'], 'dataEncoding' => 'base64', 'data' => base64_encode($data)));
$progress->update(1);
}
}
/**
* Analyze the library, generating the file and symbol maps.
*
* @return void
*/
private function analyzeLibrary()
{
// Identify all the ".php" source files in the library.
$this->log(pht('Finding source files...'));
$source_map = $this->loadSourceFileMap();
$this->log(pht('Found %s files.', new PhutilNumber(count($source_map))));
// Load the symbol cache with existing parsed symbols. This allows us
// to remap libraries quickly by analyzing only changed files.
$this->log(pht('Loading symbol cache...'));
$symbol_cache = $this->loadSymbolCache();
// If the XHPAST binary is not up-to-date, build it now. Otherwise,
// `phutil_symbols.php` will attempt to build the binary and will fail
// miserably because it will be trying to build the same file multiple
// times in parallel.
if (!PhutilXHPASTBinary::isAvailable()) {
PhutilXHPASTBinary::build();
}
// Build out the symbol analysis for all the files in the library. For
// each file, check if it's in cache. If we miss in the cache, do a fresh
// analysis.
$symbol_map = array();
$futures = array();
foreach ($source_map as $file => $hash) {
if (!empty($symbol_cache[$hash])) {
$symbol_map[$file] = $symbol_cache[$hash];
continue;
}
$futures[$file] = $this->buildSymbolAnalysisFuture($file);
}
$this->log(pht('Found %s files in cache.', new PhutilNumber(count($symbol_map))));
// Run the analyzer on any files which need analysis.
if ($futures) {
$limit = $this->subprocessLimit;
$count = number_format(count($futures));
$this->log(pht('Analyzing %d files with %d subprocesses...', $count, $limit));
$progress = new PhutilConsoleProgressBar();
if ($this->quiet) {
$progress->setQuiet(true);
}
$progress->setTotal(count($futures));
$futures = id(new FutureIterator($futures))->limit($limit);
foreach ($futures as $file => $future) {
$result = $future->resolveJSON();
if (empty($result['error'])) {
$symbol_map[$file] = $result;
} else {
$progress->done(false);
throw new XHPASTSyntaxErrorException($result['line'], $file . ': ' . $result['error']);
}
$progress->update(1);
}
$progress->done();
}
$this->fileSymbolMap = $symbol_map;
// We're done building the cache, so write it out immediately. Note that
// we've only retained entries for files we found, so this implicitly cleans
// out old cache entries.
$this->writeSymbolCache($symbol_map, $source_map);
// Our map is up to date, so either show it on stdout or write it to disk.
$this->log(pht('Building library map...'));
$this->librarySymbolMap = $this->buildLibraryMap($symbol_map);
}
/**
* Analyze the library, generating the file and symbol maps.
*
* @return void
*/
private function analyzeLibrary()
{
// Identify all the ".php" source files in the library.
$this->log("Finding source files...\n");
$source_map = $this->loadSourceFileMap();
$this->log("Found " . number_format(count($source_map)) . " files.\n");
// Load the symbol cache with existing parsed symbols. This allows us
// to remap libraries quickly by analyzing only changed files.
$this->log("Loading symbol cache...\n");
$symbol_cache = $this->loadSymbolCache();
// Build out the symbol analysis for all the files in the library. For
// each file, check if it's in cache. If we miss in the cache, do a fresh
// analysis.
$symbol_map = array();
$futures = array();
foreach ($source_map as $file => $hash) {
if (!empty($symbol_cache[$hash])) {
$symbol_map[$file] = $symbol_cache[$hash];
continue;
}
$futures[$file] = $this->buildSymbolAnalysisFuture($file);
}
$this->log("Found " . number_format(count($symbol_map)) . " files in cache.\n");
// Run the analyzer on any files which need analysis.
if ($futures) {
$limit = $this->subprocessLimit;
$count = number_format(count($futures));
$this->log("Analyzing {$count} files with {$limit} subprocesses...\n");
$progress = new PhutilConsoleProgressBar();
if ($this->quiet) {
$progress->setQuiet(true);
}
$progress->setTotal(count($futures));
foreach (Futures($futures)->limit($limit) as $file => $future) {
$result = $future->resolveJSON();
if (empty($result['error'])) {
$symbol_map[$file] = $result;
} else {
$progress->done(false);
throw new XHPASTSyntaxErrorException($result['line'], $file . ': ' . $result['error']);
}
$progress->update(1);
}
$progress->done();
}
$this->fileSymbolMap = $symbol_map;
// We're done building the cache, so write it out immediately. Note that
// we've only retained entries for files we found, so this implicitly cleans
// out old cache entries.
$this->writeSymbolCache($symbol_map, $source_map);
// Our map is up to date, so either show it on stdout or write it to disk.
$this->log("Building library map...\n");
$this->librarySymbolMap = $this->buildLibraryMap($symbol_map);
}
if (!$repository) {
throw new Exception("No repository with callsign '{$callsign}'!");
}
$commit = id(new PhabricatorRepositoryCommit())->loadOneWhere('repositoryID = %d AND commitIdentifier = %s', $repository->getID(), $commit_identifier);
if (!$commit) {
throw new Exception("No matching commit '{$commit_identifier}' in repository " . "'{$callsign}'. (For git and mercurial repositories, you must specify " . "the entire commit hash.)");
}
$commits[] = $commit;
}
}
if ($all_from_repo && !$force_local) {
echo phutil_console_format('**NOTE**: This script will queue tasks to reparse the data. Once the ' . 'tasks have been queued, you need to run Taskmaster daemons to execute ' . 'them.');
echo "\n\n";
echo "QUEUEING TASKS (" . number_format(count($commits)) . " Commits):\n";
}
$progress = new PhutilConsoleProgressBar();
$progress->setTotal(count($commits));
$tasks = array();
foreach ($commits as $commit) {
$classes = array();
switch ($repository->getVersionControlSystem()) {
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
if ($reparse_message) {
$classes[] = 'PhabricatorRepositoryGitCommitMessageParserWorker';
}
if ($reparse_change) {
$classes[] = 'PhabricatorRepositoryGitCommitChangeParserWorker';
}
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
if ($reparse_message) {
public function execute(PhutilArgumentParser $args)
{
$console = PhutilConsole::getConsole();
$all_from_repo = $args->getArg('all');
$reparse_message = $args->getArg('message');
$reparse_change = $args->getArg('change');
$reparse_herald = $args->getArg('herald');
$reparse_owners = $args->getArg('owners');
$reparse_what = $args->getArg('revision');
$force = $args->getArg('force');
$force_local = $args->getArg('force-local');
$min_date = $args->getArg('min-date');
$importing = $args->getArg('importing');
if (!$all_from_repo && !$reparse_what) {
throw new PhutilArgumentUsageException(pht('Specify a commit or repository to reparse.'));
}
if ($all_from_repo && $reparse_what) {
$commits = implode(', ', $reparse_what);
throw new PhutilArgumentUsageException(pht("Specify a commit or repository to reparse, not both:\n" . "All from repo: %s\n" . "Commit(s) to reparse: %s", $all_from_repo, $commits));
}
$any_step = $reparse_message || $reparse_change || $reparse_herald || $reparse_owners;
if ($any_step && $importing) {
throw new PhutilArgumentUsageException(pht('Choosing steps with %s conflicts with flags which select ' . 'specific steps.', '--importing'));
} else {
if ($any_step) {
// OK.
} else {
if ($importing) {
// OK.
} else {
if (!$any_step && !$importing) {
throw new PhutilArgumentUsageException(pht('Specify which steps to reparse with %s, or %s, %s, %s, or %s.', '--importing', '--message', '--change', '--herald', '--owners'));
}
}
}
}
$min_timestamp = false;
if ($min_date) {
$min_timestamp = strtotime($min_date);
if (!$all_from_repo) {
throw new PhutilArgumentUsageException(pht("You must use --all if you specify --min-date\n" . "e.g.\n" . " repository reparse --all TEST --owners --min-date yesterday"));
}
// previous to PHP 5.1.0 you would compare with -1, instead of false
if (false === $min_timestamp) {
throw new PhutilArgumentUsageException(pht("Supplied --min-date is not valid. See help for valid examples.\n" . "Supplied value: '%s'\n", $min_date));
}
}
if ($reparse_owners && !$force) {
$console->writeOut("%s\n", pht('You are about to recreate the relationship entries between the ' . 'commits and the packages they touch. This might delete some ' . 'existing relationship entries for some old commits.'));
if (!phutil_console_confirm(pht('Are you ready to continue?'))) {
throw new PhutilArgumentUsageException(pht('Cancelled.'));
}
}
$commits = array();
if ($all_from_repo) {
$repository = id(new PhabricatorRepositoryQuery())->setViewer(PhabricatorUser::getOmnipotentUser())->withIdentifiers(array($all_from_repo))->executeOne();
if (!$repository) {
throw new PhutilArgumentUsageException(pht('Unknown repository "%s"!', $all_from_repo));
}
$query = id(new DiffusionCommitQuery())->setViewer(PhabricatorUser::getOmnipotentUser())->withRepository($repository);
if ($min_timestamp) {
$query->withEpochRange($min_timestamp, null);
}
if ($importing) {
$query->withImporting(true);
}
$commits = $query->execute();
if (!$commits) {
throw new PhutilArgumentUsageException(pht('No commits have been discovered in the "%s" repository!', $repository->getDisplayName()));
}
} else {
$commits = $this->loadNamedCommits($reparse_what);
}
if ($all_from_repo && !$force_local) {
$console->writeOut("%s\n", pht("**NOTE**: This script will queue tasks to reparse the data. Once the " . "tasks have been queued, you need to run Taskmaster daemons to " . "execute them.\n\n%s", pht('QUEUEING TASKS (%s Commit(s)):', phutil_count($commits))));
}
$progress = new PhutilConsoleProgressBar();
$progress->setTotal(count($commits));
$tasks = array();
foreach ($commits as $commit) {
$repository = $commit->getRepository();
if ($importing) {
$status = $commit->getImportStatus();
// Find the first missing import step and queue that up.
$reparse_message = false;
$reparse_change = false;
$reparse_owners = false;
$reparse_herald = false;
if (!($status & PhabricatorRepositoryCommit::IMPORTED_MESSAGE)) {
$reparse_message = true;
} else {
if (!($status & PhabricatorRepositoryCommit::IMPORTED_CHANGE)) {
$reparse_change = true;
} else {
if (!($status & PhabricatorRepositoryCommit::IMPORTED_OWNERS)) {
$reparse_owners = true;
} else {
if (!($status & PhabricatorRepositoryCommit::IMPORTED_HERALD)) {
$reparse_herald = true;
} else {
continue;
}
}
}
}
}
$classes = array();
switch ($repository->getVersionControlSystem()) {
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
if ($reparse_message) {
$classes[] = 'PhabricatorRepositoryGitCommitMessageParserWorker';
}
if ($reparse_change) {
$classes[] = 'PhabricatorRepositoryGitCommitChangeParserWorker';
}
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
if ($reparse_message) {
$classes[] = 'PhabricatorRepositoryMercurialCommitMessageParserWorker';
}
if ($reparse_change) {
$classes[] = 'PhabricatorRepositoryMercurialCommitChangeParserWorker';
}
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
if ($reparse_message) {
$classes[] = 'PhabricatorRepositorySvnCommitMessageParserWorker';
}
if ($reparse_change) {
$classes[] = 'PhabricatorRepositorySvnCommitChangeParserWorker';
}
break;
}
if ($reparse_herald) {
$classes[] = 'PhabricatorRepositoryCommitHeraldWorker';
}
if ($reparse_owners) {
$classes[] = 'PhabricatorRepositoryCommitOwnersWorker';
}
// NOTE: With "--importing", we queue the first unparsed step and let
// it queue the other ones normally. Without "--importing", we queue
// all the requested steps explicitly.
$spec = array('commitID' => $commit->getID(), 'only' => !$importing, 'forceAutoclose' => $args->getArg('force-autoclose'));
if ($all_from_repo && !$force_local) {
foreach ($classes as $class) {
PhabricatorWorker::scheduleTask($class, $spec, array('priority' => PhabricatorWorker::PRIORITY_IMPORT));
}
} else {
foreach ($classes as $class) {
$worker = newv($class, array($spec));
$worker->executeTask();
}
}
$progress->update(1);
}
$progress->done();
return 0;
}
