public function getHighlightFuture($source) { $scrub = false; if (strpos($source, '<?') === false) { $source = "<?php\n" . $source . "\n"; $scrub = true; } return new PhutilXHPASTSyntaxHighlighterFuture(PhutilXHPASTBinary::getParserFuture($source), $source, $scrub); }
/** * Build futures on this linter, for use and to share with other linters. * * @param list<string> Paths to build futures for. * @return list<ExecFuture> Futures. * @task sharing */ protected final function buildSharedFutures(array $paths) { foreach ($paths as $path) { if (!isset($this->futures[$path])) { $this->futures[$path] = PhutilXHPASTBinary::getParserFuture($this->getData($path)); $this->refcount[$path] = 1; } else { $this->refcount[$path]++; } } return array_select_keys($this->futures, $paths); }
private function extractFiles($root_path, array $files) { $hashes = array(); $futures = array(); foreach ($files as $file => $hash) { $full_path = $root_path . DIRECTORY_SEPARATOR . $file; $data = Filesystem::readFile($full_path); $futures[$full_path] = PhutilXHPASTBinary::getParserFuture($data); $hashes[$full_path] = $hash; } $bar = id(new PhutilConsoleProgressBar())->setTotal(count($futures)); $messages = array(); $results = array(); $futures = id(new FutureIterator($futures))->limit(8); foreach ($futures as $full_path => $future) { $bar->update(1); $hash = $hashes[$full_path]; try { $tree = XHPASTTree::newFromDataAndResolvedExecFuture(Filesystem::readFile($full_path), $future->resolve()); } catch (Exception $ex) { $messages[] = pht('WARNING: Failed to extract strings from file "%s": %s', $full_path, $ex->getMessage()); continue; } $root = $tree->getRootNode(); $calls = $root->selectDescendantsOfType('n_FUNCTION_CALL'); foreach ($calls as $call) { $name = $call->getChildByIndex(0)->getConcreteString(); if ($name != 'pht') { continue; } $params = $call->getChildByIndex(1, 'n_CALL_PARAMETER_LIST'); $string_node = $params->getChildByIndex(0); $string_line = $string_node->getLineNumber(); try { $string_value = $string_node->evalStatic(); $results[$hash][] = array('string' => $string_value, 'file' => Filesystem::readablePath($full_path, $root_path), 'line' => $string_line); } catch (Exception $ex) { $messages[] = pht('WARNING: Failed to evaluate pht() call on line %d in "%s": %s', $call->getLineNumber(), $full_path, $ex->getMessage()); } } $tree->dispose(); } $bar->done(); foreach ($messages as $message) { echo tsprintf("**<bg:yellow> %s </bg>** %s\n", pht('WARNING'), $message); } return $results; }
public function scan() { $output = $this->output; if (!\PhutilXHPASTBinary::isAvailable()) { $output->writeln(\PhutilXHPASTBinary::getBuildInstructions(), $output::OUTPUT_NORMAL); exit; } $base_path = $this->basePath; $php_files = $this->allFiles($base_path); $total_files = count($php_files); $this->output->writeln("{$total_files} files to scan"); /** @type \Generator $split */ $split = $this->chunk($php_files, ceil($total_files / $this->workers)); foreach ($split as $chunk) { $this->forkProcess($chunk); } foreach ($this->processes as $pID) { pcntl_waitpid($pID, $status); } }
public function handleRequest(AphrontRequest $request) { $viewer = $this->getViewer(); if ($request->isFormPost()) { $source = $request->getStr('source'); $future = PhutilXHPASTBinary::getParserFuture($source); $resolved = $future->resolve(); // This is just to let it throw exceptions if stuff is broken. try { XHPASTTree::newFromDataAndResolvedExecFuture($source, $resolved); } catch (XHPASTSyntaxErrorException $ex) { // This is possibly expected. } list($err, $stdout, $stderr) = $resolved; $storage_tree = id(new PhabricatorXHPASTParseTree())->setInput($source)->setReturnCode($err)->setStdout($stdout)->setStderr($stderr)->setAuthorPHID($viewer->getPHID())->save(); return id(new AphrontRedirectResponse())->setURI('/xhpast/view/' . $storage_tree->getID() . '/'); } $form = id(new AphrontFormView())->setUser($viewer)->appendChild(id(new AphrontFormTextAreaControl())->setLabel(pht('Source'))->setName('source')->setValue("<?php\n\n")->setHeight(AphrontFormTextAreaControl::HEIGHT_VERY_TALL))->appendChild(id(new AphrontFormSubmitControl())->setValue(pht('Parse'))); $form_box = id(new PHUIObjectBoxView())->setHeaderText(pht('Generate XHP AST'))->setForm($form); return $this->buildApplicationPage($form_box, array('title' => pht('XHPAST View'))); }
public function getHighlightFuture($language, $source) { if ($language === null) { $language = PhutilLanguageGuesser::guessLanguage($source); } $have_pygments = !empty($this->config['pygments.enabled']); if ($language == 'php' && PhutilXHPASTBinary::isAvailable()) { return id(new PhutilXHPASTSyntaxHighlighter())->getHighlightFuture($source); } if ($language == 'console') { return id(new PhutilConsoleSyntaxHighlighter())->getHighlightFuture($source); } if ($language == 'diviner' || $language == 'remarkup') { return id(new PhutilDivinerSyntaxHighlighter())->getHighlightFuture($source); } if ($language == 'rainbow') { return id(new PhutilRainbowSyntaxHighlighter())->getHighlightFuture($source); } if ($language == 'php') { return id(new PhutilLexerSyntaxHighlighter())->setConfig('lexer', new PhutilPHPFragmentLexer())->setConfig('language', 'php')->getHighlightFuture($source); } if ($language == 'py') { return id(new PhutilLexerSyntaxHighlighter())->setConfig('lexer', new PhutilPythonFragmentLexer())->setConfig('language', 'py')->getHighlightFuture($source); } if ($language == 'json') { return id(new PhutilLexerSyntaxHighlighter())->setConfig('lexer', new PhutilJSONFragmentLexer())->getHighlightFuture($source); } if ($language == 'invisible') { return id(new PhutilInvisibleSyntaxHighlighter())->getHighlightFuture($source); } // Don't invoke Pygments for plain text, since it's expensive and has // no effect. if ($language !== 'text' && $language !== 'txt') { if ($have_pygments) { return id(new PhutilPygmentsSyntaxHighlighter())->setConfig('language', $language)->getHighlightFuture($source); } } return id(new PhutilDefaultSyntaxHighlighter())->getHighlightFuture($source); }
/** * Returns the XHPAST version. * * @return string */ public static function getVersion() { if (self::$version === null) { $bin = self::getPath(); if (Filesystem::pathExists($bin)) { list($err, $stdout) = exec_manual('%s --version', $bin); if (!$err) { self::$version = trim($stdout); } } } return self::$version; }
/** * Analyze the library, generating the file and symbol maps. * * @return void */ private function analyzeLibrary() { // Identify all the ".php" source files in the library. $this->log(pht('Finding source files...')); $source_map = $this->loadSourceFileMap(); $this->log(pht('Found %s files.', new PhutilNumber(count($source_map)))); // Load the symbol cache with existing parsed symbols. This allows us // to remap libraries quickly by analyzing only changed files. $this->log(pht('Loading symbol cache...')); $symbol_cache = $this->loadSymbolCache(); // If the XHPAST binary is not up-to-date, build it now. Otherwise, // `phutil_symbols.php` will attempt to build the binary and will fail // miserably because it will be trying to build the same file multiple // times in parallel. if (!PhutilXHPASTBinary::isAvailable()) { PhutilXHPASTBinary::build(); } // Build out the symbol analysis for all the files in the library. For // each file, check if it's in cache. If we miss in the cache, do a fresh // analysis. $symbol_map = array(); $futures = array(); foreach ($source_map as $file => $hash) { if (!empty($symbol_cache[$hash])) { $symbol_map[$file] = $symbol_cache[$hash]; continue; } $futures[$file] = $this->buildSymbolAnalysisFuture($file); } $this->log(pht('Found %s files in cache.', new PhutilNumber(count($symbol_map)))); // Run the analyzer on any files which need analysis. if ($futures) { $limit = $this->subprocessLimit; $count = number_format(count($futures)); $this->log(pht('Analyzing %d files with %d subprocesses...', $count, $limit)); $progress = new PhutilConsoleProgressBar(); if ($this->quiet) { $progress->setQuiet(true); } $progress->setTotal(count($futures)); $futures = id(new FutureIterator($futures))->limit($limit); foreach ($futures as $file => $future) { $result = $future->resolveJSON(); if (empty($result['error'])) { $symbol_map[$file] = $result; } else { $progress->done(false); throw new XHPASTSyntaxErrorException($result['line'], $file . ': ' . $result['error']); } $progress->update(1); } $progress->done(); } $this->fileSymbolMap = $symbol_map; // We're done building the cache, so write it out immediately. Note that // we've only retained entries for files we found, so this implicitly cleans // out old cache entries. $this->writeSymbolCache($symbol_map, $source_map); // Our map is up to date, so either show it on stdout or write it to disk. $this->log(pht('Building library map...')); $this->librarySymbolMap = $this->buildLibraryMap($symbol_map); }
protected function executeAtomize($file_name, $file_data) { $future = PhutilXHPASTBinary::getParserFuture($file_data); $tree = XHPASTTree::newFromDataAndResolvedExecFuture($file_data, $future->resolve()); $atoms = array(); $root = $tree->getRootNode(); $func_decl = $root->selectDescendantsOfType('n_FUNCTION_DECLARATION'); foreach ($func_decl as $func) { $name = $func->getChildByIndex(2); // Don't atomize closures if ($name->getTypeName() === 'n_EMPTY') { continue; } $atom = $this->newAtom(DivinerAtom::TYPE_FUNCTION)->setName($name->getConcreteString())->setLine($func->getLineNumber())->setFile($file_name); $this->findAtomDocblock($atom, $func); $this->parseParams($atom, $func); $this->parseReturnType($atom, $func); $atoms[] = $atom; } $class_types = array(DivinerAtom::TYPE_CLASS => 'n_CLASS_DECLARATION', DivinerAtom::TYPE_INTERFACE => 'n_INTERFACE_DECLARATION'); foreach ($class_types as $atom_type => $node_type) { $class_decls = $root->selectDescendantsOfType($node_type); foreach ($class_decls as $class) { $name = $class->getChildByIndex(1, 'n_CLASS_NAME'); $atom = $this->newAtom($atom_type)->setName($name->getConcreteString())->setFile($file_name)->setLine($class->getLineNumber()); // This parses `final` and `abstract`. $attributes = $class->getChildByIndex(0, 'n_CLASS_ATTRIBUTES'); foreach ($attributes->selectDescendantsOfType('n_STRING') as $attr) { $atom->setProperty($attr->getConcreteString(), true); } // If this exists, it is `n_EXTENDS_LIST`. $extends = $class->getChildByIndex(2); $extends_class = $extends->selectDescendantsOfType('n_CLASS_NAME'); foreach ($extends_class as $parent_class) { $atom->addExtends($this->newRef(DivinerAtom::TYPE_CLASS, $parent_class->getConcreteString())); } // If this exists, it is `n_IMPLEMENTS_LIST`. $implements = $class->getChildByIndex(3); $iface_names = $implements->selectDescendantsOfType('n_CLASS_NAME'); foreach ($iface_names as $iface_name) { $atom->addExtends($this->newRef(DivinerAtom::TYPE_INTERFACE, $iface_name->getConcreteString())); } $this->findAtomDocblock($atom, $class); $methods = $class->selectDescendantsOfType('n_METHOD_DECLARATION'); foreach ($methods as $method) { $matom = $this->newAtom(DivinerAtom::TYPE_METHOD); $this->findAtomDocblock($matom, $method); $attribute_list = $method->getChildByIndex(0); $attributes = $attribute_list->selectDescendantsOfType('n_STRING'); if ($attributes) { foreach ($attributes as $attribute) { $attr = strtolower($attribute->getConcreteString()); switch ($attr) { case 'final': case 'abstract': case 'static': $matom->setProperty($attr, true); break; case 'public': case 'protected': case 'private': $matom->setProperty('access', $attr); break; } } } else { $matom->setProperty('access', 'public'); } $this->parseParams($matom, $method); $matom->setName($method->getChildByIndex(2)->getConcreteString()); $matom->setLine($method->getLineNumber()); $matom->setFile($file_name); $this->parseReturnType($matom, $method); $atom->addChild($matom); $atoms[] = $matom; } $atoms[] = $atom; } } return $atoms; }
Generate repository symbols using XHPAST. Paths are read from stdin. EOSYNOPSIS ); $args->parseStandardArguments(); if (posix_isatty(STDIN)) { echo phutil_console_format("%s\n", pht('Usage: %s', "find . -type f -name '*.php' | ./generate_php_symbols.php")); exit(1); } $input = file_get_contents('php://stdin'); $data = array(); $futures = array(); foreach (explode("\n", trim($input)) as $file) { $file = Filesystem::readablePath($file); $data[$file] = Filesystem::readFile($file); $futures[$file] = PhutilXHPASTBinary::getParserFuture($data[$file]); } $futures = new FutureIterator($futures); foreach ($futures->limit(8) as $file => $future) { $tree = XHPASTTree::newFromDataAndResolvedExecFuture($data[$file], $future->resolve()); $root = $tree->getRootNode(); $scopes = array(); $functions = $root->selectDescendantsOfType('n_FUNCTION_DECLARATION'); foreach ($functions as $function) { $name = $function->getChildByIndex(2); // Skip anonymous functions. if (!$name->getConcreteString()) { continue; } print_symbol($file, 'function', $name); }
private function executeParserTest($name, $data) { $data = explode("\n", $data, 2); if (count($data) !== 2) { throw new Exception(pht('Expected multiple lines in parser test file "%s".', $name)); } $head = head($data); $body = last($data); if (!preg_match('/^#/', $head)) { throw new Exception(pht('Expected first line of parser test file "%s" to begin with "#" ' . 'and specify test options.', $name)); } $head = preg_replace('/^#\\s*/', '', $head); $options_parser = new PhutilSimpleOptions(); $options = $options_parser->parse($head); $type = null; foreach ($options as $key => $value) { switch ($key) { case 'pass': case 'fail-syntax': case 'fail-parse': if ($type !== null) { throw new Exception(pht('Test file "%s" unexpectedly specifies multiple expected ' . 'test outcomes.', $name)); } $type = $key; break; case 'comment': // Human readable comment providing test case information. break; case 'rtrim': // Allows construction of tests which rely on EOF without newlines. $body = rtrim($body); break; default: throw new Exception(pht('Test file "%s" has unknown option "%s" in its options ' . 'string.', $name, $key)); } } if ($type === null) { throw new Exception(pht('Test file "%s" does not specify a test result (like "pass") in ' . 'its options string.', $name)); } $future = PhutilXHPASTBinary::getParserFuture($body); list($err, $stdout, $stderr) = $future->resolve(); switch ($type) { case 'pass': case 'fail-parse': $this->assertEqual(0, $err, pht('Exit code for "%s".', $name)); $expect_name = preg_replace('/\\.test$/', '.expect', $name); $dir = dirname(__FILE__) . '/data/'; $expect = Filesystem::readFile($dir . $expect_name); try { $expect = phutil_json_decode($expect); } catch (PhutilJSONParserException $ex) { throw new PhutilProxyException(pht('Test ".expect" file "%s" for test "%s" is not valid JSON.', $expect_name, $name), $ex); } try { $stdout = phutil_json_decode($stdout); } catch (PhutilJSONParserException $ex) { throw new PhutilProxyException(pht('Output for test file "%s" is not valid JSON.', $name), $ex); } $json = new PhutilJSON(); $expect_nice = $json->encodeFormatted($expect); $stdout_nice = $json->encodeFormatted($stdout); if ($type == 'pass') { $this->assertEqual($expect_nice, $stdout_nice, pht('Parser output for "%s".', $name)); } else { $this->assertFalse($expect_nice == $stdout_nice, pht('Expected parser to parse "%s" incorrectly.', $name)); } break; case 'fail-syntax': $this->assertEqual(1, $err, pht('Exit code for "%s".', $name)); $this->assertTrue((bool) preg_match('/syntax error/', $stderr), pht('Expect "syntax error" in stderr or "%s".', $name)); break; } }
#!/usr/bin/env php <?php require_once dirname(__FILE__) . '/__init_script__.php'; PhutilXHPASTBinary::build(); echo pht('Build successful!') . "\n";
private function executeParserTest($name, $file) { $contents = Filesystem::readFile($file); $contents = preg_split('/^~{4,}\\n/m', $contents); if (count($contents) < 2) { throw new Exception(pht("Expected '%s' separating test case and results.", '~~~~~~~~~~')); } list($data, $options, $expect) = array_merge($contents, array(null)); $options = id(new PhutilSimpleOptions())->parse($options); $type = null; foreach ($options as $key => $value) { switch ($key) { case 'pass': case 'fail-syntax': case 'fail-parse': if ($type !== null) { throw new Exception(pht('Test file "%s" unexpectedly specifies multiple expected ' . 'test outcomes.', $name)); } $type = $key; break; case 'comment': // Human readable comment providing test case information. break; case 'rtrim': // Allows construction of tests which rely on EOF without newlines. $data = rtrim($data); break; default: throw new Exception(pht('Test file "%s" has unknown option "%s" in its options ' . 'string.', $name, $key)); } } if ($type === null) { throw new Exception(pht('Test file "%s" does not specify a test result (like "pass") in ' . 'its options string.', $name)); } $future = PhutilXHPASTBinary::getParserFuture($data); list($err, $stdout, $stderr) = $future->resolve(); switch ($type) { case 'pass': case 'fail-parse': $this->assertEqual(0, $err, pht('Exit code for "%s".', $name)); if (!strlen($expect)) { // If there's no "expect" data in the test case, that's OK. break; } try { $expect = phutil_json_decode($expect); } catch (PhutilJSONParserException $ex) { throw new PhutilProxyException(pht('Expect data for test "%s" is not valid JSON.', $name), $ex); } try { $stdout = phutil_json_decode($stdout); } catch (PhutilJSONParserException $ex) { throw new PhutilProxyException(pht('Output for test file "%s" is not valid JSON.', $name), $ex); } $json = new PhutilJSON(); $expect_nice = $json->encodeFormatted($expect); $stdout_nice = $json->encodeFormatted($stdout); if ($type == 'pass') { $this->assertEqual($expect_nice, $stdout_nice, pht('Parser output for "%s".', $name)); } else { $this->assertFalse($expect_nice == $stdout_nice, pht('Expected parser to parse "%s" incorrectly.', $name)); } break; case 'fail-syntax': $this->assertEqual(1, $err, pht('Exit code for "%s".', $name)); $this->assertTrue((bool) preg_match('/syntax error/', $stderr), pht('Expect "syntax error" in stderr or "%s".', $name)); break; } }
public static function newFromData($php_source) { $future = PhutilXHPASTBinary::getParserFuture($php_source); return self::newFromDataAndResolvedExecFuture($php_source, $future->resolve()); }