/**
* Tokenizes the content of the source file with {@link token_get_all()} and
* filters this token stream.
*
* @return void
*/
private function _tokenize()
{
$this->tokens = array();
$this->index = 0;
$this->count = 0;
// Replace short open tags, short open tags will produce invalid results
// in all environments with disabled short open tags.
$source = $this->sourceFile->getSource();
$source = preg_replace(array('(<\\?=)', '(<\\?(\\s))'), array('<?php echo ', '<?php\\1'), $source);
if (version_compare(phpversion(), '5.3.0alpha3') < 0) {
$tokens = PHP_Depend_Tokenizer_PHP52Helper::tokenize($source);
} else {
$tokens = token_get_all($source);
}
$tokens = $this->_substituteTokens($tokens);
// Is the current token between an opening and a closing php tag?
$inTag = false;
// The current line number
$startLine = 1;
$startColumn = 1;
$endColumn = 1;
$literalMap = self::$literalMap;
$tokenMap = self::$tokenMap;
// Previous found type
$previousType = null;
while ($token = current($tokens)) {
$type = null;
$image = null;
if (is_string($token)) {
$token = array(null, $token);
}
if ($token[0] === T_OPEN_TAG) {
$type = $tokenMap[$token[0]];
$image = $token[1];
$inTag = true;
} else {
if ($token[0] === T_CLOSE_TAG) {
$type = $tokenMap[$token[0]];
$image = $token[1];
$inTag = false;
} else {
if ($inTag === false) {
$type = self::T_NO_PHP;
$image = $this->_consumeNonePhpTokens($tokens);
} else {
if ($token[0] === T_WHITESPACE) {
// Count newlines in token
$lines = substr_count($token[1], "\n");
if ($lines === 0) {
$startColumn += strlen($token[1]);
} else {
$startColumn = strlen(substr($token[1], strrpos($token[1], "\n") + 1)) + 1;
}
$startLine += $lines;
} else {
$value = strtolower($token[1]);
if (isset($literalMap[$value])) {
// Fetch literal type
$type = $literalMap[$value];
// Check for a context sensitive alternative
if (isset(self::$alternativeMap[$type][$previousType])) {
$type = self::$alternativeMap[$type][$previousType];
}
$image = $token[1];
} else {
if (isset($tokenMap[$token[0]])) {
$type = $tokenMap[$token[0]];
// Check for a context sensitive alternative
if (isset(self::$alternativeMap[$type][$previousType])) {
$type = self::$alternativeMap[$type][$previousType];
}
$image = $token[1];
} else {
// This should never happen
// @codeCoverageIgnoreStart
list($type, $image) = $this->_generateUnknownToken($token[1]);
// @codeCoverageIgnoreEnd
}
}
}
}
}
}
if ($type) {
$rtrim = rtrim($image);
$lines = substr_count($rtrim, "\n");
if ($lines === 0) {
$endColumn = $startColumn + strlen($rtrim) - 1;
} else {
$endColumn = strlen(substr($rtrim, strrpos($rtrim, "\n") + 1));
}
$endLine = $startLine + $lines;
$token = new PHP_Depend_Token($type, $rtrim, $startLine, $endLine, $startColumn, $endColumn);
// Store token in internal list
$this->tokens[] = $token;
// Count newlines in token
$lines = substr_count($image, "\n");
if ($lines === 0) {
$startColumn += strlen($image);
} else {
$startColumn = strlen(substr($image, strrpos($image, "\n") + 1)) + 1;
}
$startLine += $lines;
// Store current type
if ($type !== self::T_COMMENT && $type !== self::T_DOC_COMMENT) {
$previousType = $type;
}
}
next($tokens);
}
$this->count = count($this->tokens);
}
/**
* Tests that the helper ignores a class name in a string.
*
* @return void
* @covers PHP_Depend_Tokenizer_PHP52Helper
* @group pdepend
* @group pdepend::tokenizer
* @group unittest
*/
public function testHelperIgnoresQualifiedClassNameInDoubleQuotedString()
{
$tokens = PHP_Depend_Tokenizer_PHP52Helper::tokenize('<?php $clazz = "foo\\bar";');
$this->assertSame(array(array(T_OPEN_TAG, '<?php ', 1), array(T_VARIABLE, '$clazz', 1), array(T_WHITESPACE, ' ', 1), '=', array(T_WHITESPACE, ' ', 1), array(T_CONSTANT_ENCAPSED_STRING, '"foo\\bar"', 1), ';'), $this->appendLineNumberInPHP522($tokens));
}
/**
* testHelperThrowsExpectedExceptionWhenTokenGetAllFails
*
* @return void
* @expectedException PHP_Depend_Parser_TokenException
*/
public function testHelperThrowsExpectedExceptionWhenTokenGetAllFails()
{
if (version_compare(phpversion(), '5.3.0') >= 0) {
$this->markTestSkipped('This test only works with PHP versions < 5.2.0');
}
PHP_Depend_Tokenizer_PHP52Helper::tokenize('<?php
class MyClass {
/*
}
?>');
}
