public function getHighlightFuture($source)
 {
     $scrub = false;
     if (strpos($source, '<?') === false) {
         $source = "<?php\n" . $source . "\n";
         $scrub = true;
     }
     return new PhutilXHPASTSyntaxHighlighterFuture(PhutilXHPASTBinary::getParserFuture($source), $source, $scrub);
 }
 /**
  * Build futures on this linter, for use and to share with other linters.
  *
  * @param list<string> Paths to build futures for.
  * @return list<ExecFuture> Futures.
  * @task sharing
  */
 protected final function buildSharedFutures(array $paths)
 {
     foreach ($paths as $path) {
         if (!isset($this->futures[$path])) {
             $this->futures[$path] = PhutilXHPASTBinary::getParserFuture($this->getData($path));
             $this->refcount[$path] = 1;
         } else {
             $this->refcount[$path]++;
         }
     }
     return array_select_keys($this->futures, $paths);
 }
 private function extractFiles($root_path, array $files)
 {
     $hashes = array();
     $futures = array();
     foreach ($files as $file => $hash) {
         $full_path = $root_path . DIRECTORY_SEPARATOR . $file;
         $data = Filesystem::readFile($full_path);
         $futures[$full_path] = PhutilXHPASTBinary::getParserFuture($data);
         $hashes[$full_path] = $hash;
     }
     $bar = id(new PhutilConsoleProgressBar())->setTotal(count($futures));
     $messages = array();
     $results = array();
     $futures = id(new FutureIterator($futures))->limit(8);
     foreach ($futures as $full_path => $future) {
         $bar->update(1);
         $hash = $hashes[$full_path];
         try {
             $tree = XHPASTTree::newFromDataAndResolvedExecFuture(Filesystem::readFile($full_path), $future->resolve());
         } catch (Exception $ex) {
             $messages[] = pht('WARNING: Failed to extract strings from file "%s": %s', $full_path, $ex->getMessage());
             continue;
         }
         $root = $tree->getRootNode();
         $calls = $root->selectDescendantsOfType('n_FUNCTION_CALL');
         foreach ($calls as $call) {
             $name = $call->getChildByIndex(0)->getConcreteString();
             if ($name != 'pht') {
                 continue;
             }
             $params = $call->getChildByIndex(1, 'n_CALL_PARAMETER_LIST');
             $string_node = $params->getChildByIndex(0);
             $string_line = $string_node->getLineNumber();
             try {
                 $string_value = $string_node->evalStatic();
                 $results[$hash][] = array('string' => $string_value, 'file' => Filesystem::readablePath($full_path, $root_path), 'line' => $string_line);
             } catch (Exception $ex) {
                 $messages[] = pht('WARNING: Failed to evaluate pht() call on line %d in "%s": %s', $call->getLineNumber(), $full_path, $ex->getMessage());
             }
         }
         $tree->dispose();
     }
     $bar->done();
     foreach ($messages as $message) {
         echo tsprintf("**<bg:yellow> %s </bg>** %s\n", pht('WARNING'), $message);
     }
     return $results;
 }
Example #4
0
 public function scan()
 {
     $output = $this->output;
     if (!\PhutilXHPASTBinary::isAvailable()) {
         $output->writeln(\PhutilXHPASTBinary::getBuildInstructions(), $output::OUTPUT_NORMAL);
         exit;
     }
     $base_path = $this->basePath;
     $php_files = $this->allFiles($base_path);
     $total_files = count($php_files);
     $this->output->writeln("{$total_files} files to scan");
     /** @type \Generator $split */
     $split = $this->chunk($php_files, ceil($total_files / $this->workers));
     foreach ($split as $chunk) {
         $this->forkProcess($chunk);
     }
     foreach ($this->processes as $pID) {
         pcntl_waitpid($pID, $status);
     }
 }
 public function handleRequest(AphrontRequest $request)
 {
     $viewer = $this->getViewer();
     if ($request->isFormPost()) {
         $source = $request->getStr('source');
         $future = PhutilXHPASTBinary::getParserFuture($source);
         $resolved = $future->resolve();
         // This is just to let it throw exceptions if stuff is broken.
         try {
             XHPASTTree::newFromDataAndResolvedExecFuture($source, $resolved);
         } catch (XHPASTSyntaxErrorException $ex) {
             // This is possibly expected.
         }
         list($err, $stdout, $stderr) = $resolved;
         $storage_tree = id(new PhabricatorXHPASTParseTree())->setInput($source)->setReturnCode($err)->setStdout($stdout)->setStderr($stderr)->setAuthorPHID($viewer->getPHID())->save();
         return id(new AphrontRedirectResponse())->setURI('/xhpast/view/' . $storage_tree->getID() . '/');
     }
     $form = id(new AphrontFormView())->setUser($viewer)->appendChild(id(new AphrontFormTextAreaControl())->setLabel(pht('Source'))->setName('source')->setValue("<?php\n\n")->setHeight(AphrontFormTextAreaControl::HEIGHT_VERY_TALL))->appendChild(id(new AphrontFormSubmitControl())->setValue(pht('Parse')));
     $form_box = id(new PHUIObjectBoxView())->setHeaderText(pht('Generate XHP AST'))->setForm($form);
     return $this->buildApplicationPage($form_box, array('title' => pht('XHPAST View')));
 }
 public function getHighlightFuture($language, $source)
 {
     if ($language === null) {
         $language = PhutilLanguageGuesser::guessLanguage($source);
     }
     $have_pygments = !empty($this->config['pygments.enabled']);
     if ($language == 'php' && PhutilXHPASTBinary::isAvailable()) {
         return id(new PhutilXHPASTSyntaxHighlighter())->getHighlightFuture($source);
     }
     if ($language == 'console') {
         return id(new PhutilConsoleSyntaxHighlighter())->getHighlightFuture($source);
     }
     if ($language == 'diviner' || $language == 'remarkup') {
         return id(new PhutilDivinerSyntaxHighlighter())->getHighlightFuture($source);
     }
     if ($language == 'rainbow') {
         return id(new PhutilRainbowSyntaxHighlighter())->getHighlightFuture($source);
     }
     if ($language == 'php') {
         return id(new PhutilLexerSyntaxHighlighter())->setConfig('lexer', new PhutilPHPFragmentLexer())->setConfig('language', 'php')->getHighlightFuture($source);
     }
     if ($language == 'py') {
         return id(new PhutilLexerSyntaxHighlighter())->setConfig('lexer', new PhutilPythonFragmentLexer())->setConfig('language', 'py')->getHighlightFuture($source);
     }
     if ($language == 'json') {
         return id(new PhutilLexerSyntaxHighlighter())->setConfig('lexer', new PhutilJSONFragmentLexer())->getHighlightFuture($source);
     }
     if ($language == 'invisible') {
         return id(new PhutilInvisibleSyntaxHighlighter())->getHighlightFuture($source);
     }
     // Don't invoke Pygments for plain text, since it's expensive and has
     // no effect.
     if ($language !== 'text' && $language !== 'txt') {
         if ($have_pygments) {
             return id(new PhutilPygmentsSyntaxHighlighter())->setConfig('language', $language)->getHighlightFuture($source);
         }
     }
     return id(new PhutilDefaultSyntaxHighlighter())->getHighlightFuture($source);
 }
 /**
  * Returns the XHPAST version.
  *
  * @return string
  */
 public static function getVersion()
 {
     if (self::$version === null) {
         $bin = self::getPath();
         if (Filesystem::pathExists($bin)) {
             list($err, $stdout) = exec_manual('%s --version', $bin);
             if (!$err) {
                 self::$version = trim($stdout);
             }
         }
     }
     return self::$version;
 }
 /**
  * Analyze the library, generating the file and symbol maps.
  *
  * @return void
  */
 private function analyzeLibrary()
 {
     // Identify all the ".php" source files in the library.
     $this->log(pht('Finding source files...'));
     $source_map = $this->loadSourceFileMap();
     $this->log(pht('Found %s files.', new PhutilNumber(count($source_map))));
     // Load the symbol cache with existing parsed symbols. This allows us
     // to remap libraries quickly by analyzing only changed files.
     $this->log(pht('Loading symbol cache...'));
     $symbol_cache = $this->loadSymbolCache();
     // If the XHPAST binary is not up-to-date, build it now. Otherwise,
     // `phutil_symbols.php` will attempt to build the binary and will fail
     // miserably because it will be trying to build the same file multiple
     // times in parallel.
     if (!PhutilXHPASTBinary::isAvailable()) {
         PhutilXHPASTBinary::build();
     }
     // Build out the symbol analysis for all the files in the library. For
     // each file, check if it's in cache. If we miss in the cache, do a fresh
     // analysis.
     $symbol_map = array();
     $futures = array();
     foreach ($source_map as $file => $hash) {
         if (!empty($symbol_cache[$hash])) {
             $symbol_map[$file] = $symbol_cache[$hash];
             continue;
         }
         $futures[$file] = $this->buildSymbolAnalysisFuture($file);
     }
     $this->log(pht('Found %s files in cache.', new PhutilNumber(count($symbol_map))));
     // Run the analyzer on any files which need analysis.
     if ($futures) {
         $limit = $this->subprocessLimit;
         $count = number_format(count($futures));
         $this->log(pht('Analyzing %d files with %d subprocesses...', $count, $limit));
         $progress = new PhutilConsoleProgressBar();
         if ($this->quiet) {
             $progress->setQuiet(true);
         }
         $progress->setTotal(count($futures));
         $futures = id(new FutureIterator($futures))->limit($limit);
         foreach ($futures as $file => $future) {
             $result = $future->resolveJSON();
             if (empty($result['error'])) {
                 $symbol_map[$file] = $result;
             } else {
                 $progress->done(false);
                 throw new XHPASTSyntaxErrorException($result['line'], $file . ': ' . $result['error']);
             }
             $progress->update(1);
         }
         $progress->done();
     }
     $this->fileSymbolMap = $symbol_map;
     // We're done building the cache, so write it out immediately. Note that
     // we've only retained entries for files we found, so this implicitly cleans
     // out old cache entries.
     $this->writeSymbolCache($symbol_map, $source_map);
     // Our map is up to date, so either show it on stdout or write it to disk.
     $this->log(pht('Building library map...'));
     $this->librarySymbolMap = $this->buildLibraryMap($symbol_map);
 }
 protected function executeAtomize($file_name, $file_data)
 {
     $future = PhutilXHPASTBinary::getParserFuture($file_data);
     $tree = XHPASTTree::newFromDataAndResolvedExecFuture($file_data, $future->resolve());
     $atoms = array();
     $root = $tree->getRootNode();
     $func_decl = $root->selectDescendantsOfType('n_FUNCTION_DECLARATION');
     foreach ($func_decl as $func) {
         $name = $func->getChildByIndex(2);
         // Don't atomize closures
         if ($name->getTypeName() === 'n_EMPTY') {
             continue;
         }
         $atom = $this->newAtom(DivinerAtom::TYPE_FUNCTION)->setName($name->getConcreteString())->setLine($func->getLineNumber())->setFile($file_name);
         $this->findAtomDocblock($atom, $func);
         $this->parseParams($atom, $func);
         $this->parseReturnType($atom, $func);
         $atoms[] = $atom;
     }
     $class_types = array(DivinerAtom::TYPE_CLASS => 'n_CLASS_DECLARATION', DivinerAtom::TYPE_INTERFACE => 'n_INTERFACE_DECLARATION');
     foreach ($class_types as $atom_type => $node_type) {
         $class_decls = $root->selectDescendantsOfType($node_type);
         foreach ($class_decls as $class) {
             $name = $class->getChildByIndex(1, 'n_CLASS_NAME');
             $atom = $this->newAtom($atom_type)->setName($name->getConcreteString())->setFile($file_name)->setLine($class->getLineNumber());
             // This parses `final` and `abstract`.
             $attributes = $class->getChildByIndex(0, 'n_CLASS_ATTRIBUTES');
             foreach ($attributes->selectDescendantsOfType('n_STRING') as $attr) {
                 $atom->setProperty($attr->getConcreteString(), true);
             }
             // If this exists, it is `n_EXTENDS_LIST`.
             $extends = $class->getChildByIndex(2);
             $extends_class = $extends->selectDescendantsOfType('n_CLASS_NAME');
             foreach ($extends_class as $parent_class) {
                 $atom->addExtends($this->newRef(DivinerAtom::TYPE_CLASS, $parent_class->getConcreteString()));
             }
             // If this exists, it is `n_IMPLEMENTS_LIST`.
             $implements = $class->getChildByIndex(3);
             $iface_names = $implements->selectDescendantsOfType('n_CLASS_NAME');
             foreach ($iface_names as $iface_name) {
                 $atom->addExtends($this->newRef(DivinerAtom::TYPE_INTERFACE, $iface_name->getConcreteString()));
             }
             $this->findAtomDocblock($atom, $class);
             $methods = $class->selectDescendantsOfType('n_METHOD_DECLARATION');
             foreach ($methods as $method) {
                 $matom = $this->newAtom(DivinerAtom::TYPE_METHOD);
                 $this->findAtomDocblock($matom, $method);
                 $attribute_list = $method->getChildByIndex(0);
                 $attributes = $attribute_list->selectDescendantsOfType('n_STRING');
                 if ($attributes) {
                     foreach ($attributes as $attribute) {
                         $attr = strtolower($attribute->getConcreteString());
                         switch ($attr) {
                             case 'final':
                             case 'abstract':
                             case 'static':
                                 $matom->setProperty($attr, true);
                                 break;
                             case 'public':
                             case 'protected':
                             case 'private':
                                 $matom->setProperty('access', $attr);
                                 break;
                         }
                     }
                 } else {
                     $matom->setProperty('access', 'public');
                 }
                 $this->parseParams($matom, $method);
                 $matom->setName($method->getChildByIndex(2)->getConcreteString());
                 $matom->setLine($method->getLineNumber());
                 $matom->setFile($file_name);
                 $this->parseReturnType($matom, $method);
                 $atom->addChild($matom);
                 $atoms[] = $matom;
             }
             $atoms[] = $atom;
         }
     }
     return $atoms;
 }
  Generate repository symbols using XHPAST. Paths are read from stdin.
EOSYNOPSIS
);
$args->parseStandardArguments();
if (posix_isatty(STDIN)) {
    echo phutil_console_format("%s\n", pht('Usage: %s', "find . -type f -name '*.php' | ./generate_php_symbols.php"));
    exit(1);
}
$input = file_get_contents('php://stdin');
$data = array();
$futures = array();
foreach (explode("\n", trim($input)) as $file) {
    $file = Filesystem::readablePath($file);
    $data[$file] = Filesystem::readFile($file);
    $futures[$file] = PhutilXHPASTBinary::getParserFuture($data[$file]);
}
$futures = new FutureIterator($futures);
foreach ($futures->limit(8) as $file => $future) {
    $tree = XHPASTTree::newFromDataAndResolvedExecFuture($data[$file], $future->resolve());
    $root = $tree->getRootNode();
    $scopes = array();
    $functions = $root->selectDescendantsOfType('n_FUNCTION_DECLARATION');
    foreach ($functions as $function) {
        $name = $function->getChildByIndex(2);
        // Skip anonymous functions.
        if (!$name->getConcreteString()) {
            continue;
        }
        print_symbol($file, 'function', $name);
    }
 private function executeParserTest($name, $data)
 {
     $data = explode("\n", $data, 2);
     if (count($data) !== 2) {
         throw new Exception(pht('Expected multiple lines in parser test file "%s".', $name));
     }
     $head = head($data);
     $body = last($data);
     if (!preg_match('/^#/', $head)) {
         throw new Exception(pht('Expected first line of parser test file "%s" to begin with "#" ' . 'and specify test options.', $name));
     }
     $head = preg_replace('/^#\\s*/', '', $head);
     $options_parser = new PhutilSimpleOptions();
     $options = $options_parser->parse($head);
     $type = null;
     foreach ($options as $key => $value) {
         switch ($key) {
             case 'pass':
             case 'fail-syntax':
             case 'fail-parse':
                 if ($type !== null) {
                     throw new Exception(pht('Test file "%s" unexpectedly specifies multiple expected ' . 'test outcomes.', $name));
                 }
                 $type = $key;
                 break;
             case 'comment':
                 // Human readable comment providing test case information.
                 break;
             case 'rtrim':
                 // Allows construction of tests which rely on EOF without newlines.
                 $body = rtrim($body);
                 break;
             default:
                 throw new Exception(pht('Test file "%s" has unknown option "%s" in its options ' . 'string.', $name, $key));
         }
     }
     if ($type === null) {
         throw new Exception(pht('Test file "%s" does not specify a test result (like "pass") in ' . 'its options string.', $name));
     }
     $future = PhutilXHPASTBinary::getParserFuture($body);
     list($err, $stdout, $stderr) = $future->resolve();
     switch ($type) {
         case 'pass':
         case 'fail-parse':
             $this->assertEqual(0, $err, pht('Exit code for "%s".', $name));
             $expect_name = preg_replace('/\\.test$/', '.expect', $name);
             $dir = dirname(__FILE__) . '/data/';
             $expect = Filesystem::readFile($dir . $expect_name);
             try {
                 $expect = phutil_json_decode($expect);
             } catch (PhutilJSONParserException $ex) {
                 throw new PhutilProxyException(pht('Test ".expect" file "%s" for test "%s" is not valid JSON.', $expect_name, $name), $ex);
             }
             try {
                 $stdout = phutil_json_decode($stdout);
             } catch (PhutilJSONParserException $ex) {
                 throw new PhutilProxyException(pht('Output for test file "%s" is not valid JSON.', $name), $ex);
             }
             $json = new PhutilJSON();
             $expect_nice = $json->encodeFormatted($expect);
             $stdout_nice = $json->encodeFormatted($stdout);
             if ($type == 'pass') {
                 $this->assertEqual($expect_nice, $stdout_nice, pht('Parser output for "%s".', $name));
             } else {
                 $this->assertFalse($expect_nice == $stdout_nice, pht('Expected parser to parse "%s" incorrectly.', $name));
             }
             break;
         case 'fail-syntax':
             $this->assertEqual(1, $err, pht('Exit code for "%s".', $name));
             $this->assertTrue((bool) preg_match('/syntax error/', $stderr), pht('Expect "syntax error" in stderr or "%s".', $name));
             break;
     }
 }
#!/usr/bin/env php
<?php 
require_once dirname(__FILE__) . '/__init_script__.php';
PhutilXHPASTBinary::build();
echo pht('Build successful!') . "\n";
 private function executeParserTest($name, $file)
 {
     $contents = Filesystem::readFile($file);
     $contents = preg_split('/^~{4,}\\n/m', $contents);
     if (count($contents) < 2) {
         throw new Exception(pht("Expected '%s' separating test case and results.", '~~~~~~~~~~'));
     }
     list($data, $options, $expect) = array_merge($contents, array(null));
     $options = id(new PhutilSimpleOptions())->parse($options);
     $type = null;
     foreach ($options as $key => $value) {
         switch ($key) {
             case 'pass':
             case 'fail-syntax':
             case 'fail-parse':
                 if ($type !== null) {
                     throw new Exception(pht('Test file "%s" unexpectedly specifies multiple expected ' . 'test outcomes.', $name));
                 }
                 $type = $key;
                 break;
             case 'comment':
                 // Human readable comment providing test case information.
                 break;
             case 'rtrim':
                 // Allows construction of tests which rely on EOF without newlines.
                 $data = rtrim($data);
                 break;
             default:
                 throw new Exception(pht('Test file "%s" has unknown option "%s" in its options ' . 'string.', $name, $key));
         }
     }
     if ($type === null) {
         throw new Exception(pht('Test file "%s" does not specify a test result (like "pass") in ' . 'its options string.', $name));
     }
     $future = PhutilXHPASTBinary::getParserFuture($data);
     list($err, $stdout, $stderr) = $future->resolve();
     switch ($type) {
         case 'pass':
         case 'fail-parse':
             $this->assertEqual(0, $err, pht('Exit code for "%s".', $name));
             if (!strlen($expect)) {
                 // If there's no "expect" data in the test case, that's OK.
                 break;
             }
             try {
                 $expect = phutil_json_decode($expect);
             } catch (PhutilJSONParserException $ex) {
                 throw new PhutilProxyException(pht('Expect data for test "%s" is not valid JSON.', $name), $ex);
             }
             try {
                 $stdout = phutil_json_decode($stdout);
             } catch (PhutilJSONParserException $ex) {
                 throw new PhutilProxyException(pht('Output for test file "%s" is not valid JSON.', $name), $ex);
             }
             $json = new PhutilJSON();
             $expect_nice = $json->encodeFormatted($expect);
             $stdout_nice = $json->encodeFormatted($stdout);
             if ($type == 'pass') {
                 $this->assertEqual($expect_nice, $stdout_nice, pht('Parser output for "%s".', $name));
             } else {
                 $this->assertFalse($expect_nice == $stdout_nice, pht('Expected parser to parse "%s" incorrectly.', $name));
             }
             break;
         case 'fail-syntax':
             $this->assertEqual(1, $err, pht('Exit code for "%s".', $name));
             $this->assertTrue((bool) preg_match('/syntax error/', $stderr), pht('Expect "syntax error" in stderr or "%s".', $name));
             break;
     }
 }
Example #14
0
 public static function newFromData($php_source)
 {
     $future = PhutilXHPASTBinary::getParserFuture($php_source);
     return self::newFromDataAndResolvedExecFuture($php_source, $future->resolve());
 }