private function uploadChunks($file_phid, $path) { $conduit = $this->getConduit(); $f = @fopen($path, 'rb'); if (!$f) { throw new Exception(pht('Unable to open file "%s"', $path)); } $this->writeStatus(pht('Beginning chunked upload of large file...')); $chunks = $conduit->callMethodSynchronous('file.querychunks', array('filePHID' => $file_phid)); $remaining = array(); foreach ($chunks as $chunk) { if (!$chunk['complete']) { $remaining[] = $chunk; } } $done = count($chunks) - count($remaining); if ($done) { $this->writeStatus(pht('Resuming upload (%d of %d chunks remain).', new PhutilNumber(count($remaining)), new PhutilNumber(count($chunks)))); } else { $this->writeStatus(pht('Uploading chunks (%d chunks to upload).', new PhutilNumber(count($remaining)))); } $progress = new PhutilConsoleProgressBar(); $progress->setTotal(count($chunks)); for ($ii = 0; $ii < $done; $ii++) { $progress->update(1); } $progress->draw(); // TODO: We could do these in parallel to improve upload performance. foreach ($remaining as $chunk) { $offset = $chunk['byteStart']; $ok = fseek($f, $offset); if ($ok !== 0) { throw new Exception(pht('Failed to %s!', 'fseek()')); } $data = fread($f, $chunk['byteEnd'] - $chunk['byteStart']); if ($data === false) { throw new Exception(pht('Failed to %s!', 'fread()')); } $conduit->callMethodSynchronous('file.uploadchunk', array('filePHID' => $file_phid, 'byteStart' => $offset, 'dataEncoding' => 'base64', 'data' => base64_encode($data))); $progress->update(1); } }
/** * Upload missing chunks of a large file by calling `file.uploadchunk` over * Conduit. * * @task internal */ private function uploadChunks(ArcanistFileDataRef $file, $file_phid) { $conduit = $this->conduit; $chunks = $conduit->callMethodSynchronous('file.querychunks', array('filePHID' => $file_phid)); $remaining = array(); foreach ($chunks as $chunk) { if (!$chunk['complete']) { $remaining[] = $chunk; } } $done = count($chunks) - count($remaining); if ($done) { $this->writeStatus(pht('Resuming upload (%s of %s chunks remain).', phutil_count($remaining), phutil_count($chunks))); } else { $this->writeStatus(pht('Uploading chunks (%s chunks to upload).', phutil_count($remaining))); } $progress = new PhutilConsoleProgressBar(); $progress->setTotal(count($chunks)); for ($ii = 0; $ii < $done; $ii++) { $progress->update(1); } $progress->draw(); // TODO: We could do these in parallel to improve upload performance. foreach ($remaining as $chunk) { $data = $file->readBytes($chunk['byteStart'], $chunk['byteEnd']); $conduit->callMethodSynchronous('file.uploadchunk', array('filePHID' => $file_phid, 'byteStart' => $chunk['byteStart'], 'dataEncoding' => 'base64', 'data' => base64_encode($data))); $progress->update(1); } }
/** * Analyze the library, generating the file and symbol maps. * * @return void */ private function analyzeLibrary() { // Identify all the ".php" source files in the library. $this->log(pht('Finding source files...')); $source_map = $this->loadSourceFileMap(); $this->log(pht('Found %s files.', new PhutilNumber(count($source_map)))); // Load the symbol cache with existing parsed symbols. This allows us // to remap libraries quickly by analyzing only changed files. $this->log(pht('Loading symbol cache...')); $symbol_cache = $this->loadSymbolCache(); // If the XHPAST binary is not up-to-date, build it now. Otherwise, // `phutil_symbols.php` will attempt to build the binary and will fail // miserably because it will be trying to build the same file multiple // times in parallel. if (!PhutilXHPASTBinary::isAvailable()) { PhutilXHPASTBinary::build(); } // Build out the symbol analysis for all the files in the library. For // each file, check if it's in cache. If we miss in the cache, do a fresh // analysis. $symbol_map = array(); $futures = array(); foreach ($source_map as $file => $hash) { if (!empty($symbol_cache[$hash])) { $symbol_map[$file] = $symbol_cache[$hash]; continue; } $futures[$file] = $this->buildSymbolAnalysisFuture($file); } $this->log(pht('Found %s files in cache.', new PhutilNumber(count($symbol_map)))); // Run the analyzer on any files which need analysis. if ($futures) { $limit = $this->subprocessLimit; $count = number_format(count($futures)); $this->log(pht('Analyzing %d files with %d subprocesses...', $count, $limit)); $progress = new PhutilConsoleProgressBar(); if ($this->quiet) { $progress->setQuiet(true); } $progress->setTotal(count($futures)); $futures = id(new FutureIterator($futures))->limit($limit); foreach ($futures as $file => $future) { $result = $future->resolveJSON(); if (empty($result['error'])) { $symbol_map[$file] = $result; } else { $progress->done(false); throw new XHPASTSyntaxErrorException($result['line'], $file . ': ' . $result['error']); } $progress->update(1); } $progress->done(); } $this->fileSymbolMap = $symbol_map; // We're done building the cache, so write it out immediately. Note that // we've only retained entries for files we found, so this implicitly cleans // out old cache entries. $this->writeSymbolCache($symbol_map, $source_map); // Our map is up to date, so either show it on stdout or write it to disk. $this->log(pht('Building library map...')); $this->librarySymbolMap = $this->buildLibraryMap($symbol_map); }
/** * Analyze the library, generating the file and symbol maps. * * @return void */ private function analyzeLibrary() { // Identify all the ".php" source files in the library. $this->log("Finding source files...\n"); $source_map = $this->loadSourceFileMap(); $this->log("Found " . number_format(count($source_map)) . " files.\n"); // Load the symbol cache with existing parsed symbols. This allows us // to remap libraries quickly by analyzing only changed files. $this->log("Loading symbol cache...\n"); $symbol_cache = $this->loadSymbolCache(); // Build out the symbol analysis for all the files in the library. For // each file, check if it's in cache. If we miss in the cache, do a fresh // analysis. $symbol_map = array(); $futures = array(); foreach ($source_map as $file => $hash) { if (!empty($symbol_cache[$hash])) { $symbol_map[$file] = $symbol_cache[$hash]; continue; } $futures[$file] = $this->buildSymbolAnalysisFuture($file); } $this->log("Found " . number_format(count($symbol_map)) . " files in cache.\n"); // Run the analyzer on any files which need analysis. if ($futures) { $limit = $this->subprocessLimit; $count = number_format(count($futures)); $this->log("Analyzing {$count} files with {$limit} subprocesses...\n"); $progress = new PhutilConsoleProgressBar(); if ($this->quiet) { $progress->setQuiet(true); } $progress->setTotal(count($futures)); foreach (Futures($futures)->limit($limit) as $file => $future) { $result = $future->resolveJSON(); if (empty($result['error'])) { $symbol_map[$file] = $result; } else { $progress->done(false); throw new XHPASTSyntaxErrorException($result['line'], $file . ': ' . $result['error']); } $progress->update(1); } $progress->done(); } $this->fileSymbolMap = $symbol_map; // We're done building the cache, so write it out immediately. Note that // we've only retained entries for files we found, so this implicitly cleans // out old cache entries. $this->writeSymbolCache($symbol_map, $source_map); // Our map is up to date, so either show it on stdout or write it to disk. $this->log("Building library map...\n"); $this->librarySymbolMap = $this->buildLibraryMap($symbol_map); }
if (!$repository) { throw new Exception("No repository with callsign '{$callsign}'!"); } $commit = id(new PhabricatorRepositoryCommit())->loadOneWhere('repositoryID = %d AND commitIdentifier = %s', $repository->getID(), $commit_identifier); if (!$commit) { throw new Exception("No matching commit '{$commit_identifier}' in repository " . "'{$callsign}'. (For git and mercurial repositories, you must specify " . "the entire commit hash.)"); } $commits[] = $commit; } } if ($all_from_repo && !$force_local) { echo phutil_console_format('**NOTE**: This script will queue tasks to reparse the data. Once the ' . 'tasks have been queued, you need to run Taskmaster daemons to execute ' . 'them.'); echo "\n\n"; echo "QUEUEING TASKS (" . number_format(count($commits)) . " Commits):\n"; } $progress = new PhutilConsoleProgressBar(); $progress->setTotal(count($commits)); $tasks = array(); foreach ($commits as $commit) { $classes = array(); switch ($repository->getVersionControlSystem()) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: if ($reparse_message) { $classes[] = 'PhabricatorRepositoryGitCommitMessageParserWorker'; } if ($reparse_change) { $classes[] = 'PhabricatorRepositoryGitCommitChangeParserWorker'; } break; case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: if ($reparse_message) {
public function execute(PhutilArgumentParser $args) { $console = PhutilConsole::getConsole(); $all_from_repo = $args->getArg('all'); $reparse_message = $args->getArg('message'); $reparse_change = $args->getArg('change'); $reparse_herald = $args->getArg('herald'); $reparse_owners = $args->getArg('owners'); $reparse_what = $args->getArg('revision'); $force = $args->getArg('force'); $force_local = $args->getArg('force-local'); $min_date = $args->getArg('min-date'); $importing = $args->getArg('importing'); if (!$all_from_repo && !$reparse_what) { throw new PhutilArgumentUsageException(pht('Specify a commit or repository to reparse.')); } if ($all_from_repo && $reparse_what) { $commits = implode(', ', $reparse_what); throw new PhutilArgumentUsageException(pht("Specify a commit or repository to reparse, not both:\n" . "All from repo: %s\n" . "Commit(s) to reparse: %s", $all_from_repo, $commits)); } $any_step = $reparse_message || $reparse_change || $reparse_herald || $reparse_owners; if ($any_step && $importing) { throw new PhutilArgumentUsageException(pht('Choosing steps with %s conflicts with flags which select ' . 'specific steps.', '--importing')); } else { if ($any_step) { // OK. } else { if ($importing) { // OK. } else { if (!$any_step && !$importing) { throw new PhutilArgumentUsageException(pht('Specify which steps to reparse with %s, or %s, %s, %s, or %s.', '--importing', '--message', '--change', '--herald', '--owners')); } } } } $min_timestamp = false; if ($min_date) { $min_timestamp = strtotime($min_date); if (!$all_from_repo) { throw new PhutilArgumentUsageException(pht("You must use --all if you specify --min-date\n" . "e.g.\n" . " repository reparse --all TEST --owners --min-date yesterday")); } // previous to PHP 5.1.0 you would compare with -1, instead of false if (false === $min_timestamp) { throw new PhutilArgumentUsageException(pht("Supplied --min-date is not valid. See help for valid examples.\n" . "Supplied value: '%s'\n", $min_date)); } } if ($reparse_owners && !$force) { $console->writeOut("%s\n", pht('You are about to recreate the relationship entries between the ' . 'commits and the packages they touch. This might delete some ' . 'existing relationship entries for some old commits.')); if (!phutil_console_confirm(pht('Are you ready to continue?'))) { throw new PhutilArgumentUsageException(pht('Cancelled.')); } } $commits = array(); if ($all_from_repo) { $repository = id(new PhabricatorRepositoryQuery())->setViewer(PhabricatorUser::getOmnipotentUser())->withIdentifiers(array($all_from_repo))->executeOne(); if (!$repository) { throw new PhutilArgumentUsageException(pht('Unknown repository "%s"!', $all_from_repo)); } $query = id(new DiffusionCommitQuery())->setViewer(PhabricatorUser::getOmnipotentUser())->withRepository($repository); if ($min_timestamp) { $query->withEpochRange($min_timestamp, null); } if ($importing) { $query->withImporting(true); } $commits = $query->execute(); if (!$commits) { throw new PhutilArgumentUsageException(pht('No commits have been discovered in the "%s" repository!', $repository->getDisplayName())); } } else { $commits = $this->loadNamedCommits($reparse_what); } if ($all_from_repo && !$force_local) { $console->writeOut("%s\n", pht("**NOTE**: This script will queue tasks to reparse the data. Once the " . "tasks have been queued, you need to run Taskmaster daemons to " . "execute them.\n\n%s", pht('QUEUEING TASKS (%s Commit(s)):', phutil_count($commits)))); } $progress = new PhutilConsoleProgressBar(); $progress->setTotal(count($commits)); $tasks = array(); foreach ($commits as $commit) { $repository = $commit->getRepository(); if ($importing) { $status = $commit->getImportStatus(); // Find the first missing import step and queue that up. $reparse_message = false; $reparse_change = false; $reparse_owners = false; $reparse_herald = false; if (!($status & PhabricatorRepositoryCommit::IMPORTED_MESSAGE)) { $reparse_message = true; } else { if (!($status & PhabricatorRepositoryCommit::IMPORTED_CHANGE)) { $reparse_change = true; } else { if (!($status & PhabricatorRepositoryCommit::IMPORTED_OWNERS)) { $reparse_owners = true; } else { if (!($status & PhabricatorRepositoryCommit::IMPORTED_HERALD)) { $reparse_herald = true; } else { continue; } } } } } $classes = array(); switch ($repository->getVersionControlSystem()) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: if ($reparse_message) { $classes[] = 'PhabricatorRepositoryGitCommitMessageParserWorker'; } if ($reparse_change) { $classes[] = 'PhabricatorRepositoryGitCommitChangeParserWorker'; } break; case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: if ($reparse_message) { $classes[] = 'PhabricatorRepositoryMercurialCommitMessageParserWorker'; } if ($reparse_change) { $classes[] = 'PhabricatorRepositoryMercurialCommitChangeParserWorker'; } break; case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: if ($reparse_message) { $classes[] = 'PhabricatorRepositorySvnCommitMessageParserWorker'; } if ($reparse_change) { $classes[] = 'PhabricatorRepositorySvnCommitChangeParserWorker'; } break; } if ($reparse_herald) { $classes[] = 'PhabricatorRepositoryCommitHeraldWorker'; } if ($reparse_owners) { $classes[] = 'PhabricatorRepositoryCommitOwnersWorker'; } // NOTE: With "--importing", we queue the first unparsed step and let // it queue the other ones normally. Without "--importing", we queue // all the requested steps explicitly. $spec = array('commitID' => $commit->getID(), 'only' => !$importing, 'forceAutoclose' => $args->getArg('force-autoclose')); if ($all_from_repo && !$force_local) { foreach ($classes as $class) { PhabricatorWorker::scheduleTask($class, $spec, array('priority' => PhabricatorWorker::PRIORITY_IMPORT)); } } else { foreach ($classes as $class) { $worker = newv($class, array($spec)); $worker->executeTask(); } } $progress->update(1); } $progress->done(); return 0; }