Splits the provided $code into PHP language tokens.
public static tokenize ( string $code, array $options = [] ) : array | ||
$code | string | Source code to be tokenized. |
$options | array | Options consists of: -'wrap': Boolean indicating whether or not to wrap the supplied code in PHP tags. -'ignore': An array containing PHP language tokens to ignore. -'include': If supplied, an array of the only language tokens to include in the output. |
Résultat | array | An array of tokens in the supplied source code. |
public function testFilteredTokenization() { $code = 'while (isset($countRugen)) { if ($inigoMontoya->is("alive")) { ' . "\n"; $code .= '$inigoMontoya->say(array("hello", "name", "accusation", "die")); ' . "\n"; $code .= 'try { $inigoMontoya->kill($countRugen); } catch (Exception $e) { continue; } } }'; $result = Parser::tokenize($code, array('include' => array('T_IF', 'T_WHILE', 'T_CATCH'))); $expected = array(array('id' => 318, 'name' => 'T_WHILE', 'content' => 'while', 'line' => 1), array('id' => 301, 'name' => 'T_IF', 'content' => 'if', 'line' => 1), array('id' => 338, 'name' => 'T_CATCH', 'content' => 'catch', 'line' => 3)); $this->assertEqual($expected, $result); }
/** * Takes an instance of an object (usually a Collection object) containing test * instances. Introspects the test subject classes to extract cyclomatic complexity data. * * @param object $report Instance of Report which is calling apply. * @param array $tests The test to apply this filter on * @param array $options Not used. * @return object|void Returns the instance of `$tests`. */ public static function apply($report, $tests, array $options = array()) { $results = array(); foreach ($tests->invoke('subject') as $class) { $results[$class] = array(); if (!($methods = Inspector::methods($class, 'ranges', array('public' => false)))) { continue; } foreach ($methods as $method => $lines) { $lines = Inspector::lines($class, $lines); $branches = Parser::tokenize(join("\n", (array) $lines), array('include' => static::$_include)); $results[$class][$method] = count($branches) + 1; $report->collect(__CLASS__, $results); } } return $tests; }
protected function _display($file) { if (!($matches = Parser::tokenize(file_get_contents($file)))) { $this->stop(1, 'no matches'); } if (!$this->show) { $this->out($file . ':'); } $rows = array(array('', 'ID', 'LINE', 'TYPE', 'TEXT')); foreach ($matches as $match) { if (($id = substr(sha1($file . $match['line']), 0, 4)) == $this->show) { $this->stop(0, $file); } $rows[] = array('', $id, $match['line'], $match['type'], $match['text']); } if (!$this->show) { $this->out($this->columns($rows)); $this->hr(); $this->nl(); } }
public function testParserGuessesLineBleed() { $code = <<<EOD if (false) { \treturn true; } EOD; $tokens = Parser::tokenize($code); $this->assertIdentical('}', $tokens[13]['content']); $this->assertIdentical(3, $tokens[13]['line']); }
/** * Adds more token information than the base lithium tokenizer such as name, * parent, and children. * * @param string $code Source code to be tokenized. * @param array $options Options consists of: * -'wrap': Boolean indicating whether or not to wrap the supplied * code in PHP tags. * -'ignore': An array containing PHP language tokens to ignore. * -'include': If supplied, an array of the only language tokens * to include in the output. * @return array An array of extracted information from the supplied source code: * - lineCache: token ids indexed by line number * - typeCache: token ids indexed by token type * - meta: parsing information (level, etc.) indexed by token id * - relationships: parent and child relations (token ids) indexed by token id */ public static function tokenize($code, array $options = array()) { $options += array('wrap' => true); $tokens = static::_tokenize(parent::tokenize($code, $options)); static::$_bracketsChecksum = 0; $curParent = -1; $brackets = $curlyBrackets = $squareBrackets = 0; $level = $needNestUp = $nestLine = $nestLevel = 0; $nestLog = $lineCache = $typeCache = array(); $inString = false; $inPhp = $options['wrap'] ? true : false; foreach ($tokens as $tokenId => $token) { $isString = $token['id'] === T_CONSTANT_ENCAPSED_STRING || $token['id'] === T_ENCAPSED_AND_WHITESPACE; $lineCache[$token['line']][] = $tokenId; if ($isString) { $carriageReturn = substr_count($token['content'], "\n"); for ($i = 1; $i <= $carriageReturn; $i++) { $lineCache[$token['line'] + $i][] = $tokenId; } } $typeCache[$token['id']][] = $tokenId; if ($token['id'] === T_CLOSE_TAG) { $needNestUp = 0; $nestLevel = 0; $inPhp = false; } if ($token['id'] === T_OPEN_TAG) { $inPhp = true; } if (!$inPhp) { continue; } if ($token['id'] === T_END_DOUBLE_QUOTE || $token['id'] === T_END_HEREDOC) { $inString = false; } if ($needNestUp > 0) { $status = $nestLog[$needNestUp]; if (!$status['founded'] && in_array($token['content'], $status['nestOn'])) { $nestLog[$needNestUp]['founded'] = true; } if ($token['line'] > $nestLine && $status['founded'] && !$status['applied']) { $nestLevel++; $nestLog[$needNestUp]['applied'] = true; } } $tokens[$tokenId] = static::_checksum($tokens[$tokenId], $isString || $inString); $tokens[$tokenId]['level'] = $level; $tokens[$tokenId]['parent'] = $curParent; $tokens[$tokenId]['children'] = array(); if (isset($tokens[$curParent])) { $tokens[$curParent]['children'][] = $tokenId; } while (($parent = static::_isEndOfParent($tokenId, $curParent, $tokens)) !== false) { if (!$inString && static::$_parentTokens[$tokens[$curParent]['id']]['nestOn']) { $needNestUp === 0 ?: $needNestUp--; $nestLevel = $tokens[$curParent]['nestLevel']; } $level--; $curParent = $parent; } $tokens[$tokenId]['nestLevel'] = $inString ? null : $nestLevel; $tokens[$tokenId]['isString'] = $isString; if ($token['id'] === T_START_DOUBLE_QUOTE || $token['id'] === T_START_HEREDOC) { $inString = true; } if (static::_isParent($tokenId, $tokens)) { $tokens[$tokenId]['parent'] = $curParent; if (!$inString && ($nestOn = static::$_parentTokens[$token['id']]['nestOn'])) { $nestLine = $token['line']; $nestLog[++$needNestUp] = array('nestOn' => $nestOn, 'founded' => $nestOn === true ? true : false, 'applied' => false); } $curParent = $tokenId; $level++; } } if ($level !== 0 || $squareBrackets !== 0 || $curlyBrackets !== 0 || $brackets !== 0) { $smallTokens = array_slice($tokens, 0, 20); $exception = new ParserException('A parse error has been encountered.'); $exception->parserData = compact('level', 'curlyBrackets', 'brackets', 'tokens'); throw $exception; } return compact('tokens', 'lineCache', 'typeCache'); }
public function testParserGuessesLineBleedWithNonWhitespace() { $code = <<<EOD if (false) { \t// hello world } EOD; $tokens = Parser::tokenize($code); $this->assertIdentical('}', $tokens[9]['content']); $this->assertIdentical(3, $tokens[9]['line']); }