Splits the provided $code into PHP language tokens.
| public static tokenize ( string $code, array $options = [] ) : array | ||
| $code | string | Source code to be tokenized. |
| $options | array | Options consists of: -'wrap': Boolean indicating whether or not to wrap the supplied code in PHP tags. -'ignore': An array containing PHP language tokens to ignore. -'include': If supplied, an array of the only language tokens to include in the output. |
| return | array | An array of tokens in the supplied source code. |
public function testFilteredTokenization()
{
$code = 'while (isset($countRugen)) { if ($inigoMontoya->is("alive")) { ' . "\n";
$code .= '$inigoMontoya->say(array("hello", "name", "accusation", "die")); ' . "\n";
$code .= 'try { $inigoMontoya->kill($countRugen); } catch (Exception $e) { continue; } } }';
$result = Parser::tokenize($code, array('include' => array('T_IF', 'T_WHILE', 'T_CATCH')));
$expected = array(array('id' => 318, 'name' => 'T_WHILE', 'content' => 'while', 'line' => 1), array('id' => 301, 'name' => 'T_IF', 'content' => 'if', 'line' => 1), array('id' => 338, 'name' => 'T_CATCH', 'content' => 'catch', 'line' => 3));
$this->assertEqual($expected, $result);
}
/**
* Takes an instance of an object (usually a Collection object) containing test
* instances. Introspects the test subject classes to extract cyclomatic complexity data.
*
* @param object $report Instance of Report which is calling apply.
* @param array $tests The test to apply this filter on
* @param array $options Not used.
* @return object|void Returns the instance of `$tests`.
*/
public static function apply($report, $tests, array $options = array())
{
$results = array();
foreach ($tests->invoke('subject') as $class) {
$results[$class] = array();
if (!($methods = Inspector::methods($class, 'ranges', array('public' => false)))) {
continue;
}
foreach ($methods as $method => $lines) {
$lines = Inspector::lines($class, $lines);
$branches = Parser::tokenize(join("\n", (array) $lines), array('include' => static::$_include));
$results[$class][$method] = count($branches) + 1;
$report->collect(__CLASS__, $results);
}
}
return $tests;
}
protected function _display($file)
{
if (!($matches = Parser::tokenize(file_get_contents($file)))) {
$this->stop(1, 'no matches');
}
if (!$this->show) {
$this->out($file . ':');
}
$rows = array(array('', 'ID', 'LINE', 'TYPE', 'TEXT'));
foreach ($matches as $match) {
if (($id = substr(sha1($file . $match['line']), 0, 4)) == $this->show) {
$this->stop(0, $file);
}
$rows[] = array('', $id, $match['line'], $match['type'], $match['text']);
}
if (!$this->show) {
$this->out($this->columns($rows));
$this->hr();
$this->nl();
}
}
public function testParserGuessesLineBleed()
{
$code = <<<EOD
if (false) {
\treturn true;
}
EOD;
$tokens = Parser::tokenize($code);
$this->assertIdentical('}', $tokens[13]['content']);
$this->assertIdentical(3, $tokens[13]['line']);
}
/**
* Adds more token information than the base lithium tokenizer such as name,
* parent, and children.
*
* @param string $code Source code to be tokenized.
* @param array $options Options consists of:
* -'wrap': Boolean indicating whether or not to wrap the supplied
* code in PHP tags.
* -'ignore': An array containing PHP language tokens to ignore.
* -'include': If supplied, an array of the only language tokens
* to include in the output.
* @return array An array of extracted information from the supplied source code:
* - lineCache: token ids indexed by line number
* - typeCache: token ids indexed by token type
* - meta: parsing information (level, etc.) indexed by token id
* - relationships: parent and child relations (token ids) indexed by token id
*/
public static function tokenize($code, array $options = array())
{
$options += array('wrap' => true);
$tokens = static::_tokenize(parent::tokenize($code, $options));
static::$_bracketsChecksum = 0;
$curParent = -1;
$brackets = $curlyBrackets = $squareBrackets = 0;
$level = $needNestUp = $nestLine = $nestLevel = 0;
$nestLog = $lineCache = $typeCache = array();
$inString = false;
$inPhp = $options['wrap'] ? true : false;
foreach ($tokens as $tokenId => $token) {
$isString = $token['id'] === T_CONSTANT_ENCAPSED_STRING || $token['id'] === T_ENCAPSED_AND_WHITESPACE;
$lineCache[$token['line']][] = $tokenId;
if ($isString) {
$carriageReturn = substr_count($token['content'], "\n");
for ($i = 1; $i <= $carriageReturn; $i++) {
$lineCache[$token['line'] + $i][] = $tokenId;
}
}
$typeCache[$token['id']][] = $tokenId;
if ($token['id'] === T_CLOSE_TAG) {
$needNestUp = 0;
$nestLevel = 0;
$inPhp = false;
}
if ($token['id'] === T_OPEN_TAG) {
$inPhp = true;
}
if (!$inPhp) {
continue;
}
if ($token['id'] === T_END_DOUBLE_QUOTE || $token['id'] === T_END_HEREDOC) {
$inString = false;
}
if ($needNestUp > 0) {
$status = $nestLog[$needNestUp];
if (!$status['founded'] && in_array($token['content'], $status['nestOn'])) {
$nestLog[$needNestUp]['founded'] = true;
}
if ($token['line'] > $nestLine && $status['founded'] && !$status['applied']) {
$nestLevel++;
$nestLog[$needNestUp]['applied'] = true;
}
}
$tokens[$tokenId] = static::_checksum($tokens[$tokenId], $isString || $inString);
$tokens[$tokenId]['level'] = $level;
$tokens[$tokenId]['parent'] = $curParent;
$tokens[$tokenId]['children'] = array();
if (isset($tokens[$curParent])) {
$tokens[$curParent]['children'][] = $tokenId;
}
while (($parent = static::_isEndOfParent($tokenId, $curParent, $tokens)) !== false) {
if (!$inString && static::$_parentTokens[$tokens[$curParent]['id']]['nestOn']) {
$needNestUp === 0 ?: $needNestUp--;
$nestLevel = $tokens[$curParent]['nestLevel'];
}
$level--;
$curParent = $parent;
}
$tokens[$tokenId]['nestLevel'] = $inString ? null : $nestLevel;
$tokens[$tokenId]['isString'] = $isString;
if ($token['id'] === T_START_DOUBLE_QUOTE || $token['id'] === T_START_HEREDOC) {
$inString = true;
}
if (static::_isParent($tokenId, $tokens)) {
$tokens[$tokenId]['parent'] = $curParent;
if (!$inString && ($nestOn = static::$_parentTokens[$token['id']]['nestOn'])) {
$nestLine = $token['line'];
$nestLog[++$needNestUp] = array('nestOn' => $nestOn, 'founded' => $nestOn === true ? true : false, 'applied' => false);
}
$curParent = $tokenId;
$level++;
}
}
if ($level !== 0 || $squareBrackets !== 0 || $curlyBrackets !== 0 || $brackets !== 0) {
$smallTokens = array_slice($tokens, 0, 20);
$exception = new ParserException('A parse error has been encountered.');
$exception->parserData = compact('level', 'curlyBrackets', 'brackets', 'tokens');
throw $exception;
}
return compact('tokens', 'lineCache', 'typeCache');
}
public function testParserGuessesLineBleedWithNonWhitespace()
{
$code = <<<EOD
if (false) {
\t// hello world
}
EOD;
$tokens = Parser::tokenize($code);
$this->assertIdentical('}', $tokens[9]['content']);
$this->assertIdentical(3, $tokens[9]['line']);
}
