tokenize() public static method

Splits the provided $code into PHP language tokens.
public static tokenize ( string $code, array $options = [] ) : array
$code string Source code to be tokenized.
$options array Options consists of: -'wrap': Boolean indicating whether or not to wrap the supplied code in PHP tags. -'ignore': An array containing PHP language tokens to ignore. -'include': If supplied, an array of the only language tokens to include in the output.
return array An array of tokens in the supplied source code.
 public function testFilteredTokenization()
 {
     $code = 'while (isset($countRugen)) { if ($inigoMontoya->is("alive")) { ' . "\n";
     $code .= '$inigoMontoya->say(array("hello", "name", "accusation", "die")); ' . "\n";
     $code .= 'try { $inigoMontoya->kill($countRugen); } catch (Exception $e) { continue; } } }';
     $result = Parser::tokenize($code, array('include' => array('T_IF', 'T_WHILE', 'T_CATCH')));
     $expected = array(array('id' => 318, 'name' => 'T_WHILE', 'content' => 'while', 'line' => 1), array('id' => 301, 'name' => 'T_IF', 'content' => 'if', 'line' => 1), array('id' => 338, 'name' => 'T_CATCH', 'content' => 'catch', 'line' => 3));
     $this->assertEqual($expected, $result);
 }
Esempio n. 2
0
 /**
  * Takes an instance of an object (usually a Collection object) containing test
  * instances. Introspects the test subject classes to extract cyclomatic complexity data.
  *
  * @param object $report Instance of Report which is calling apply.
  * @param array $tests The test to apply this filter on
  * @param array $options Not used.
  * @return object|void Returns the instance of `$tests`.
  */
 public static function apply($report, $tests, array $options = array())
 {
     $results = array();
     foreach ($tests->invoke('subject') as $class) {
         $results[$class] = array();
         if (!($methods = Inspector::methods($class, 'ranges', array('public' => false)))) {
             continue;
         }
         foreach ($methods as $method => $lines) {
             $lines = Inspector::lines($class, $lines);
             $branches = Parser::tokenize(join("\n", (array) $lines), array('include' => static::$_include));
             $results[$class][$method] = count($branches) + 1;
             $report->collect(__CLASS__, $results);
         }
     }
     return $tests;
 }
Esempio n. 3
0
 protected function _display($file)
 {
     if (!($matches = Parser::tokenize(file_get_contents($file)))) {
         $this->stop(1, 'no matches');
     }
     if (!$this->show) {
         $this->out($file . ':');
     }
     $rows = array(array('', 'ID', 'LINE', 'TYPE', 'TEXT'));
     foreach ($matches as $match) {
         if (($id = substr(sha1($file . $match['line']), 0, 4)) == $this->show) {
             $this->stop(0, $file);
         }
         $rows[] = array('', $id, $match['line'], $match['type'], $match['text']);
     }
     if (!$this->show) {
         $this->out($this->columns($rows));
         $this->hr();
         $this->nl();
     }
 }
Esempio n. 4
0
    public function testParserGuessesLineBleed()
    {
        $code = <<<EOD
if (false) {
\treturn true;
}
EOD;
        $tokens = Parser::tokenize($code);
        $this->assertIdentical('}', $tokens[13]['content']);
        $this->assertIdentical(3, $tokens[13]['line']);
    }
Esempio n. 5
0
 /**
  * Adds more token information than the base lithium tokenizer such as name,
  * parent, and children.
  *
  * @param string $code Source code to be tokenized.
  * @param array $options Options consists of:
  *        -'wrap': Boolean indicating whether or not to wrap the supplied
  *          code in PHP tags.
  *        -'ignore': An array containing PHP language tokens to ignore.
  *        -'include': If supplied, an array of the only language tokens
  *         to include in the output.
  * @return array An array of extracted information from the supplied source code:
  *         - lineCache: token ids indexed by line number
  *         - typeCache: token ids indexed by token type
  *         - meta: parsing information (level, etc.) indexed by token id
  *         - relationships: parent and child relations (token ids) indexed by token id
  */
 public static function tokenize($code, array $options = array())
 {
     $options += array('wrap' => true);
     $tokens = static::_tokenize(parent::tokenize($code, $options));
     static::$_bracketsChecksum = 0;
     $curParent = -1;
     $brackets = $curlyBrackets = $squareBrackets = 0;
     $level = $needNestUp = $nestLine = $nestLevel = 0;
     $nestLog = $lineCache = $typeCache = array();
     $inString = false;
     $inPhp = $options['wrap'] ? true : false;
     foreach ($tokens as $tokenId => $token) {
         $isString = $token['id'] === T_CONSTANT_ENCAPSED_STRING || $token['id'] === T_ENCAPSED_AND_WHITESPACE;
         $lineCache[$token['line']][] = $tokenId;
         if ($isString) {
             $carriageReturn = substr_count($token['content'], "\n");
             for ($i = 1; $i <= $carriageReturn; $i++) {
                 $lineCache[$token['line'] + $i][] = $tokenId;
             }
         }
         $typeCache[$token['id']][] = $tokenId;
         if ($token['id'] === T_CLOSE_TAG) {
             $needNestUp = 0;
             $nestLevel = 0;
             $inPhp = false;
         }
         if ($token['id'] === T_OPEN_TAG) {
             $inPhp = true;
         }
         if (!$inPhp) {
             continue;
         }
         if ($token['id'] === T_END_DOUBLE_QUOTE || $token['id'] === T_END_HEREDOC) {
             $inString = false;
         }
         if ($needNestUp > 0) {
             $status = $nestLog[$needNestUp];
             if (!$status['founded'] && in_array($token['content'], $status['nestOn'])) {
                 $nestLog[$needNestUp]['founded'] = true;
             }
             if ($token['line'] > $nestLine && $status['founded'] && !$status['applied']) {
                 $nestLevel++;
                 $nestLog[$needNestUp]['applied'] = true;
             }
         }
         $tokens[$tokenId] = static::_checksum($tokens[$tokenId], $isString || $inString);
         $tokens[$tokenId]['level'] = $level;
         $tokens[$tokenId]['parent'] = $curParent;
         $tokens[$tokenId]['children'] = array();
         if (isset($tokens[$curParent])) {
             $tokens[$curParent]['children'][] = $tokenId;
         }
         while (($parent = static::_isEndOfParent($tokenId, $curParent, $tokens)) !== false) {
             if (!$inString && static::$_parentTokens[$tokens[$curParent]['id']]['nestOn']) {
                 $needNestUp === 0 ?: $needNestUp--;
                 $nestLevel = $tokens[$curParent]['nestLevel'];
             }
             $level--;
             $curParent = $parent;
         }
         $tokens[$tokenId]['nestLevel'] = $inString ? null : $nestLevel;
         $tokens[$tokenId]['isString'] = $isString;
         if ($token['id'] === T_START_DOUBLE_QUOTE || $token['id'] === T_START_HEREDOC) {
             $inString = true;
         }
         if (static::_isParent($tokenId, $tokens)) {
             $tokens[$tokenId]['parent'] = $curParent;
             if (!$inString && ($nestOn = static::$_parentTokens[$token['id']]['nestOn'])) {
                 $nestLine = $token['line'];
                 $nestLog[++$needNestUp] = array('nestOn' => $nestOn, 'founded' => $nestOn === true ? true : false, 'applied' => false);
             }
             $curParent = $tokenId;
             $level++;
         }
     }
     if ($level !== 0 || $squareBrackets !== 0 || $curlyBrackets !== 0 || $brackets !== 0) {
         $smallTokens = array_slice($tokens, 0, 20);
         $exception = new ParserException('A parse error has been encountered.');
         $exception->parserData = compact('level', 'curlyBrackets', 'brackets', 'tokens');
         throw $exception;
     }
     return compact('tokens', 'lineCache', 'typeCache');
 }
Esempio n. 6
0
    public function testParserGuessesLineBleedWithNonWhitespace()
    {
        $code = <<<EOD
if (false) {
\t// hello world
}
EOD;
        $tokens = Parser::tokenize($code);
        $this->assertIdentical('}', $tokens[9]['content']);
        $this->assertIdentical(3, $tokens[9]['line']);
    }