/**
* Tokenizes the content of the source file with {@link token_get_all()} and
* filters this token stream.
*
* @return void
*/
private function tokenize()
{
if ($this->tokens) {
return;
}
$this->tokens = array();
$this->index = 0;
$this->count = 0;
// Replace short open tags, short open tags will produce invalid results
// in all environments with disabled short open tags.
$source = $this->sourceFile->getSource();
$source = preg_replace(array('(<\\?=)', '(<\\?(\\s))'), array('<?php echo ', '<?php\\1'), $source);
if (version_compare(phpversion(), '5.3.0alpha3') < 0) {
$tokens = PHPTokenizerHelperVersion52::tokenize($source);
} else {
$tokens = token_get_all($source);
}
$tokens = $this->substituteTokens($tokens);
// Is the current token between an opening and a closing php tag?
$inTag = false;
// The current line number
$startLine = 1;
$startColumn = 1;
$endColumn = 1;
$literalMap = self::$literalMap;
$tokenMap = self::$tokenMap;
// Previous found type
$previousType = null;
while ($token = current($tokens)) {
$type = null;
$image = null;
if (is_string($token)) {
$token = array(null, $token);
}
if ($token[0] === T_OPEN_TAG) {
$type = $tokenMap[$token[0]];
$image = $token[1];
$inTag = true;
} elseif ($token[0] === T_CLOSE_TAG) {
$type = $tokenMap[$token[0]];
$image = $token[1];
$inTag = false;
} elseif ($inTag === false) {
$type = Tokens::T_NO_PHP;
$image = $this->consumeNonePhpTokens($tokens);
} elseif ($token[0] === T_WHITESPACE) {
// Count newlines in token
$lines = substr_count($token[1], "\n");
if ($lines === 0) {
$startColumn += strlen($token[1]);
} else {
$startColumn = strlen(substr($token[1], strrpos($token[1], "\n") + 1)) + 1;
}
$startLine += $lines;
} else {
$value = strtolower($token[1]);
if (isset($literalMap[$value])) {
// Fetch literal type
$type = $literalMap[$value];
$image = $token[1];
// Check for a context sensitive alternative
if (isset(self::$alternativeMap[$type][$previousType])) {
$type = self::$alternativeMap[$type][$previousType];
}
if (isset(self::$reductionMap[$type][$previousType])) {
$image = self::$reductionMap[$type][$previousType]['image'];
$type = self::$reductionMap[$type][$previousType]['type'];
array_pop($this->tokens);
}
} elseif (isset($tokenMap[$token[0]])) {
$type = $tokenMap[$token[0]];
// Check for a context sensitive alternative
if (isset(self::$alternativeMap[$type][$previousType])) {
$type = self::$alternativeMap[$type][$previousType];
}
$image = $token[1];
} else {
// This should never happen
// @codeCoverageIgnoreStart
list($type, $image) = $this->generateUnknownToken($token[1]);
// @codeCoverageIgnoreEnd
}
}
if ($type) {
$rtrim = rtrim($image);
$lines = substr_count($rtrim, "\n");
if ($lines === 0) {
$endColumn = $startColumn + strlen($rtrim) - 1;
} else {
$endColumn = strlen(substr($rtrim, strrpos($rtrim, "\n") + 1));
}
$endLine = $startLine + $lines;
$token = new Token($type, $rtrim, $startLine, $endLine, $startColumn, $endColumn);
// Store token in internal list
$this->tokens[] = $token;
// Count newlines in token
$lines = substr_count($image, "\n");
if ($lines === 0) {
$startColumn += strlen($image);
} else {
$startColumn = strlen(substr($image, strrpos($image, "\n") + 1)) + 1;
}
$startLine += $lines;
// Store current type
if ($type !== Tokens::T_COMMENT && $type !== Tokens::T_DOC_COMMENT) {
$previousType = $type;
}
}
next($tokens);
}
$this->count = count($this->tokens);
}
/**
* testHelperThrowsExpectedExceptionWhenTokenGetAllFails
*
* @return void
* @expectedException \PDepend\Source\Parser\TokenException
*/
public function testHelperThrowsExpectedExceptionWhenTokenGetAllFails()
{
if (version_compare(phpversion(), '5.3.0') >= 0) {
$this->markTestSkipped('This test only works with PHP versions < 5.2.0');
}
PHPTokenizerHelperVersion52::tokenize('<?php
class MyClass {
/*
}
?>');
}
