/**
* Creates an array of tokens when given some PHP code.
*
* Starts by using token_get_all() but does a lot of extra processing
* to insert information about the context of the token.
*
* @param string $string The string to tokenize.
* @param string $eolChar The EOL character to use for splitting strings.
*
* @return array
*/
public function tokenizeString($string, $eolChar = '\\n')
{
if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo "\t*** START PHP TOKENIZING ***" . PHP_EOL;
}
$tokens = @token_get_all($string);
$finalTokens = array();
$newStackPtr = 0;
$numTokens = count($tokens);
$insideInlineIf = false;
for ($stackPtr = 0; $stackPtr < $numTokens; $stackPtr++) {
$token = $tokens[$stackPtr];
$tokenIsArray = is_array($token);
if (PHP_CODESNIFFER_VERBOSITY > 1) {
if ($tokenIsArray === true) {
$type = token_name($token[0]);
$content = str_replace($eolChar, "[30;1m\\n[0m", $token[1]);
} else {
$newToken = PHP_CodeSniffer::resolveSimpleToken($token);
$type = $newToken['type'];
$content = $token;
}
$content = str_replace(' ', "[30;1m·[0m", $content);
echo "\tProcess token {$stackPtr}: {$type} => {$content}" . PHP_EOL;
}
/*
If we are using \r\n newline characters, the \r and \n are sometimes
split over two tokens. This normally occurs after comments. We need
to merge these two characters together so that our line endings are
consistent for all lines.
*/
if ($tokenIsArray === true && substr($token[1], -1) === "\r") {
if (isset($tokens[$stackPtr + 1]) === true && is_array($tokens[$stackPtr + 1]) === true && $tokens[$stackPtr + 1][1][0] === "\n") {
$token[1] .= "\n";
if ($tokens[$stackPtr + 1][1] === "\n") {
// The next token's content has been merged into this token,
// so we can skip it.
$stackPtr++;
} else {
$tokens[$stackPtr + 1][1] = substr($tokens[$stackPtr + 1][1], 1);
}
}
}
//end if
/*
If this is a double quoted string, PHP will tokenise the whole
thing which causes problems with the scope map when braces are
within the string. So we need to merge the tokens together to
provide a single string.
*/
if ($tokenIsArray === false && $token === '"') {
$tokenContent = '"';
$nestedVars = array();
for ($i = $stackPtr + 1; $i < $numTokens; $i++) {
$subTokenIsArray = is_array($tokens[$i]);
if ($subTokenIsArray === true) {
$tokenContent .= $tokens[$i][1];
if ($tokens[$i][1] === '{' && $tokens[$i][0] !== T_ENCAPSED_AND_WHITESPACE) {
$nestedVars[] = $i;
}
} else {
$tokenContent .= $tokens[$i];
if ($tokens[$i] === '}') {
array_pop($nestedVars);
}
}
if ($subTokenIsArray === false && $tokens[$i] === '"' && empty($nestedVars) === true) {
// We found the other end of the double quoted string.
break;
}
}
//end for
$stackPtr = $i;
// Convert each line within the double quoted string to a
// new token, so it conforms with other multiple line tokens.
$tokenLines = explode($eolChar, $tokenContent);
$numLines = count($tokenLines);
$newToken = array();
for ($j = 0; $j < $numLines; $j++) {
$newToken['content'] = $tokenLines[$j];
if ($j === $numLines - 1) {
if ($tokenLines[$j] === '') {
break;
}
} else {
$newToken['content'] .= $eolChar;
}
$newToken['code'] = T_DOUBLE_QUOTED_STRING;
$newToken['type'] = 'T_DOUBLE_QUOTED_STRING';
$finalTokens[$newStackPtr] = $newToken;
$newStackPtr++;
}
// Continue, as we're done with this token.
continue;
}
//end if
/*
If this is a heredoc, PHP will tokenise the whole
thing which causes problems when heredocs don't
contain real PHP code, which is almost never.
We want to leave the start and end heredoc tokens
alone though.
*/
if ($tokenIsArray === true && $token[0] === T_START_HEREDOC) {
// Add the start heredoc token to the final array.
$finalTokens[$newStackPtr] = PHP_CodeSniffer::standardiseToken($token);
// Check if this is actually a nowdoc and use a different token
// to help the sniffs.
$nowdoc = false;
if ($token[1][3] === "'") {
$finalTokens[$newStackPtr]['code'] = T_START_NOWDOC;
$finalTokens[$newStackPtr]['type'] = 'T_START_NOWDOC';
$nowdoc = true;
}
$newStackPtr++;
$tokenContent = '';
for ($i = $stackPtr + 1; $i < $numTokens; $i++) {
$subTokenIsArray = is_array($tokens[$i]);
if ($subTokenIsArray === true && $tokens[$i][0] === T_END_HEREDOC) {
// We found the other end of the heredoc.
break;
}
if ($subTokenIsArray === true) {
$tokenContent .= $tokens[$i][1];
} else {
$tokenContent .= $tokens[$i];
}
}
$stackPtr = $i;
// Convert each line within the heredoc to a
// new token, so it conforms with other multiple line tokens.
$tokenLines = explode($eolChar, $tokenContent);
$numLines = count($tokenLines);
$newToken = array();
for ($j = 0; $j < $numLines; $j++) {
$newToken['content'] = $tokenLines[$j];
if ($j === $numLines - 1) {
if ($tokenLines[$j] === '') {
break;
}
} else {
$newToken['content'] .= $eolChar;
}
if ($nowdoc === true) {
$newToken['code'] = T_NOWDOC;
$newToken['type'] = 'T_NOWDOC';
} else {
$newToken['code'] = T_HEREDOC;
$newToken['type'] = 'T_HEREDOC';
}
$finalTokens[$newStackPtr] = $newToken;
$newStackPtr++;
}
//end for
// Add the end heredoc token to the final array.
$finalTokens[$newStackPtr] = PHP_CodeSniffer::standardiseToken($tokens[$stackPtr]);
if ($nowdoc === true) {
$finalTokens[$newStackPtr]['code'] = T_END_NOWDOC;
$finalTokens[$newStackPtr]['type'] = 'T_END_NOWDOC';
$nowdoc = true;
}
$newStackPtr++;
// Continue, as we're done with this token.
continue;
}
//end if
/*
PHP doesn't assign a token to goto labels, so we have to.
These are just string tokens with a single colon after them. Double
colons are already tokenized and so don't interfere with this check.
But we do have to account for CASE statements, that look just like
goto labels.
*/
if ($tokenIsArray === true && $token[0] === T_STRING && $tokens[$stackPtr + 1] === ':' && $tokens[$stackPtr - 1][0] !== T_PAAMAYIM_NEKUDOTAYIM) {
$stopTokens = array(T_CASE, T_SEMICOLON, T_OPEN_CURLY_BRACKET, T_INLINE_THEN);
for ($x = $newStackPtr - 1; $x > 0; $x--) {
if (in_array($finalTokens[$x]['code'], $stopTokens) === true) {
break;
}
}
if ($finalTokens[$x]['code'] !== T_CASE && $finalTokens[$x]['code'] !== T_INLINE_THEN) {
$finalTokens[$newStackPtr] = array('content' => $token[1] . ':', 'code' => T_GOTO_LABEL, 'type' => 'T_GOTO_LABEL');
if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo "\t\t* token {$stackPtr} changed from T_STRING to T_GOTO_LABEL" . PHP_EOL;
echo "\t\t* skipping T_COLON token " . ($stackPtr + 1) . PHP_EOL;
}
$newStackPtr++;
$stackPtr++;
continue;
}
}
//end if
/*
If this token has newlines in its content, split each line up
and create a new token for each line. We do this so it's easier
to ascertain where errors occur on a line.
Note that $token[1] is the token's content.
*/
if ($tokenIsArray === true && strpos($token[1], $eolChar) !== false) {
$tokenLines = explode($eolChar, $token[1]);
$numLines = count($tokenLines);
$tokenName = token_name($token[0]);
for ($i = 0; $i < $numLines; $i++) {
$newToken['content'] = $tokenLines[$i];
if ($i === $numLines - 1) {
if ($tokenLines[$i] === '') {
break;
}
} else {
$newToken['content'] .= $eolChar;
}
$newToken['type'] = $tokenName;
$newToken['code'] = $token[0];
$finalTokens[$newStackPtr] = $newToken;
$newStackPtr++;
}
} else {
$newToken = PHP_CodeSniffer::standardiseToken($token);
// Convert colons that are actually the ELSE component of an
// inline IF statement.
if ($newToken['code'] === T_INLINE_THEN) {
$insideInlineIf = true;
} else {
if ($insideInlineIf === true && $newToken['code'] === T_COLON) {
$insideInlineIf = false;
$newToken['code'] = T_INLINE_ELSE;
$newToken['type'] = 'T_INLINE_ELSE';
}
}
// This is a special condition for T_ARRAY tokens used for
// type hinting function arguments as being arrays. We want to keep
// the parenthesis map clean, so let's tag these tokens as
// T_ARRAY_HINT.
if ($newToken['code'] === T_ARRAY) {
// Recalculate number of tokens.
$numTokens = count($tokens);
for ($i = $stackPtr; $i < $numTokens; $i++) {
if (is_array($tokens[$i]) === false) {
if ($tokens[$i] === '(') {
break;
}
} else {
if ($tokens[$i][0] === T_VARIABLE) {
$newToken['code'] = T_ARRAY_HINT;
$newToken['type'] = 'T_ARRAY_HINT';
break;
}
}
}
}
$finalTokens[$newStackPtr] = $newToken;
$newStackPtr++;
}
//end if
}
//end for
if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo "\t*** END PHP TOKENIZING ***" . PHP_EOL;
}
return $finalTokens;
}