public static function calculateTrailingWhitespaceIndent(Token $token)
{
if (!$token->isWhitespace()) {
throw new \InvalidArgumentException('The given token must be whitespace.');
}
return ltrim(strrchr(str_replace(array("\r\n", "\r"), "\n", $token->getContent()), 10), "\n");
}
/**
* {@inheritdoc}
*/
public function process(Tokens $tokens, Token $token, $index)
{
$prevTokenIndex = $tokens->getPrevMeaningfulToken($index);
$prevToken = $prevTokenIndex === null ? null : $tokens[$prevTokenIndex];
// Skip whole class braces content.
// That way we can skip whole tokens in class declaration, therefore skip `T_USE` for traits.
if ($token->isClassy() && !$prevToken->isGivenKind(T_DOUBLE_COLON)) {
$index = $tokens->getNextTokenOfKind($index, array('{'));
$innerLimit = $tokens->findBlockEnd(Tokens::BLOCK_TYPE_CURLY_BRACE, $index);
while ($index < $innerLimit) {
$token = $tokens[++$index];
if (!$token->isGivenKind(T_USE)) {
continue;
}
if ($this->isUseForLambda($tokens, $index)) {
$token->override(array(CT_USE_LAMBDA, $token->getContent()));
} else {
$token->override(array(CT_USE_TRAIT, $token->getContent()));
}
}
return;
}
if ($token->isGivenKind(T_USE) && $this->isUseForLambda($tokens, $index)) {
$token->override(array(CT_USE_LAMBDA, $token->getContent()));
}
}
/**
* Check if token is equals to given one.
*
* If tokens are arrays, then only keys defined in parameter token are checked.
*
* @param Token|array|string $other token or it's prototype
* @param bool $caseSensitive perform a case sensitive comparison
*
* @return bool
*/
public function equals($other, $caseSensitive = true)
{
$otherPrototype = $other instanceof Token ? $other->getPrototype() : $other;
if ($this->isArray() !== is_array($otherPrototype)) {
return false;
}
if (!$this->isArray()) {
return $this->content === $otherPrototype;
}
$selfPrototype = $this->getPrototype();
foreach ($otherPrototype as $key => $val) {
// make sure the token has such key
if (!isset($selfPrototype[$key])) {
return false;
}
if (1 === $key && !$caseSensitive) {
// case-insensitive comparison only applies to the content (key 1)
if (0 !== strcasecmp($val, $selfPrototype[1])) {
return false;
}
} else {
// regular comparison
if ($selfPrototype[$key] !== $val) {
return false;
}
}
}
return true;
}
private function fixWhitespace(Token $token)
{
$content = $token->getContent();
if (substr_count($content, "\n") > 1) {
$lines = Utils::splitLines($content);
$token->setContent("\n" . end($lines));
}
}
/**
* {@inheritdoc}
*/
public function process(Tokens $tokens, Token $token, $index)
{
if (!$this->isShortArray($tokens, $index)) {
return;
}
$endIndex = $tokens->findBlockEnd(Tokens::BLOCK_TYPE_INDEX_SQUARE_BRACE, $index);
$token->override(array(CT_ARRAY_SQUARE_BRACE_OPEN, '['));
$tokens[$endIndex]->override(array(CT_ARRAY_SQUARE_BRACE_CLOSE, ']'));
}
/**
* {@inheritdoc}
*/
public function process(Tokens $tokens, Token $token, $index)
{
if (!$token->isGivenKind(array(T_CONST, T_FUNCTION))) {
return;
}
$prevToken = $tokens[$tokens->getPrevMeaningfulToken($index)];
if ($prevToken->isGivenKind(T_USE)) {
$token->override(array($token->isGivenKind(T_FUNCTION) ? CT_FUNCTION_IMPORT : CT_CONST_IMPORT, $token->getContent()));
}
}
/**
* Cleanup a whitespace token.
*
* @param Token $token
*/
private function fixWhitespace(Token $token)
{
$content = $token->getContent();
// if there is more than one new line in the whitespace, then we need to fix it
if (substr_count($content, "\n") > 1) {
// the final bit of the whitespace must be the next statement's indentation
$lines = Utils::splitLines($content);
$token->setContent("\n" . end($lines));
}
}
/**
* {@inheritdoc}
*/
public function process(Tokens $tokens, Token $token, $index)
{
if (!$token->isGivenKind(T_CLASS)) {
return;
}
$prevIndex = $tokens->getPrevMeaningfulToken($index);
$prevToken = $tokens[$prevIndex];
if ($prevToken->isGivenKind(T_DOUBLE_COLON)) {
$token->override(array(CT_CLASS_CONSTANT, $token->getContent()));
}
}
/**
* {@inheritdoc}
*/
public function process(Tokens $tokens, Token $token, $index)
{
if (!$token->isGivenKind(T_NAMESPACE)) {
return;
}
$nextIndex = $tokens->getNextMeaningfulToken($index);
$nextToken = $tokens[$nextIndex];
if ($nextToken->equals(array(T_NS_SEPARATOR))) {
$token->override(array(CT_NAMESPACE_OPERATOR, $token->getContent()));
}
}
/**
* {@inheritdoc}
*/
public function process(Tokens $tokens, Token $token, $index)
{
if (!$token->isGivenKind(T_ARRAY)) {
return;
}
$nextIndex = $tokens->getNextMeaningfulToken($index);
$nextToken = $tokens[$nextIndex];
if (!$nextToken->equals('(')) {
$token->override(array(CT_ARRAY_TYPEHINT, $token->getContent()));
}
}
private function isValidList(Tokens $tokens, Token $docsToken, $listIndex)
{
$endIndex = $tokens->getNextTokenOfKind($listIndex, array(')'));
$docsContent = $docsToken->getContent();
for ($index = $listIndex + 1; $index < $endIndex; ++$index) {
$token = $tokens[$index];
if ($token->isGivenKind(T_VARIABLE) && false !== strpos($docsContent, $token->getContent())) {
return true;
}
}
return false;
}
public function setContent($content)
{
$currentContent = $this->getContent();
parent::setContent($content);
if ($currentContent !== $content) {
$this->messages = array();
}
}
/**
* {@inheritdoc}
*/
public function process(Tokens $tokens, Token $token, $index)
{
if (!$token->isComment()) {
return;
}
$content = $token->getContent();
$trimmedContent = rtrim($content);
// nothing trimmed, nothing to do
if ($content === $trimmedContent) {
return;
}
$whitespaces = substr($content, strlen($trimmedContent));
$token->setContent($trimmedContent);
if (isset($tokens[$index + 1]) && $tokens[$index + 1]->isGivenKind(T_WHITESPACE)) {
$tokens[$index + 1]->setContent($whitespaces . $tokens[$index + 1]->getContent());
} else {
$tokens->insertAt($index + 1, new Token(array(T_WHITESPACE, $whitespaces)));
}
}
/**
* Checks variable assignments for correct docblock usage.
*
* @param Tokens $tokens
* @param Token $docsToken docs Token
* @param int $variableIndex index of variable Token
*
* @return bool
*/
private function isValidVariable(Tokens $tokens, Token $docsToken, $variableIndex)
{
$nextIndex = $tokens->getNextMeaningfulToken($variableIndex);
if (!$tokens[$nextIndex]->equals('=')) {
return false;
}
return false !== strpos($docsToken->getContent(), $tokens[$variableIndex]->getContent());
}
/**
* If given token is a single line whitespace then fix it to be a single space.
*
* @param Token $token
*/
private function fixWhitespace(Token $token)
{
if ($token->isWhitespace(" \t")) {
$token->setContent(' ');
}
}
private function fixWhitespace(Token $token)
{
if ($token->isWhitespace(array('whitespaces' => " \t"))) {
$token->setContent(' ');
}
}
private function transformIntoDynamicVarBraces(Tokens $tokens, Token $token, $index)
{
if (!$token->equals('$')) {
return;
}
$openIndex = $tokens->getNextMeaningfulToken($index);
if (null === $openIndex) {
return;
}
$openToken = $tokens[$openIndex];
if (!$openToken->equals('{')) {
return;
}
$closeIndex = $tokens->findBlockEnd(Tokens::BLOCK_TYPE_CURLY_BRACE, $openIndex);
$closeToken = $tokens[$closeIndex];
$openToken->override(array(CT_DYNAMIC_VAR_BRACE_OPEN, '{'));
$closeToken->override(array(CT_DYNAMIC_VAR_BRACE_CLOSE, '}'));
}
/**
* Register token as found.
*
* @param Token|array|string $token token prototype
*/
private function registerFoundToken($token)
{
$tokenKind = $token instanceof Token ? $token->isArray() ? $token->getId() : $token->getContent() : (is_array($token) ? $token[0] : $token);
$this->foundTokenKinds[$tokenKind] = true;
}
/**
* Override token.
*
* If called on Token inside Tokens collection please use `Tokens::overrideAt` instead.
*
* @param Token|array|string $other token prototype
*/
public function override($other)
{
$prototype = $other instanceof self ? $other->getPrototype() : $other;
if ($this->equals($prototype)) {
return;
}
$this->changed = true;
if (is_array($prototype)) {
$this->isArray = true;
$this->id = $prototype[0];
$this->content = $prototype[1];
return;
}
$this->isArray = false;
$this->id = null;
$this->content = $prototype;
}
public function provideFindSequenceExceptions()
{
$emptyToken = new Token('!');
$emptyToken->clear();
return array(array('Invalid sequence', array()), array('Non-meaningful token at position: 0', array(array(T_WHITESPACE, ' '))), array('Non-meaningful token at position: 1', array('{', array(T_COMMENT, '// Foo'), '}')), array('Non-meaningful token at position: 2', array('{', '!', $emptyToken, '}')));
}
/**
* Find a sequence of meaningful tokens and returns the array of their locations.
*
* @param array $sequence an array of tokens (same format used by getNextTokenOfKind)
* @param int $start start index, defaulting to the start of the file
* @param int $end end index, defaulting to the end of the file
* @param bool|array $caseSensitive global case sensitiveness or an array of booleans, whose keys should match
* the ones used in $others. If any is missing, the default case-sensitive
* comparison is used
*
* @return array|null an array containing the tokens matching the sequence elements, indexed by their position
*/
public function findSequence(array $sequence, $start = 0, $end = null, $caseSensitive = true)
{
// $end defaults to the end of the collection
if (null === $end) {
$end = count($this) - 1;
}
if (!count($sequence)) {
throw new \InvalidArgumentException('Invalid sequence.');
}
// make sure the sequence content is "meaningful"
foreach ($sequence as $key => $token) {
// if not a Token instance already, we convert it to verify the meaningfulness
if (!$token instanceof Token) {
if (is_array($token) && !isset($token[1])) {
// fake some content as it is required by the Token constructor,
// although optional for search purposes
$token[1] = '';
}
$token = new Token($token);
}
if ($token->isWhitespace() || $token->isComment() || $token->isEmpty()) {
throw new \InvalidArgumentException(sprintf('Non-meaningful token at position: %s.', $key));
}
}
// remove the first token from the sequence, so we can freely iterate through the sequence after a match to
// the first one is found
$key = key($sequence);
$firstCs = Token::isKeyCaseSensitive($caseSensitive, $key);
$firstToken = $sequence[$key];
unset($sequence[$key]);
// begin searching for the first token in the sequence (start included)
$index = $start - 1;
while (null !== $index && $index <= $end) {
$index = $this->getNextTokenOfKind($index, array($firstToken), $firstCs);
// ensure we found a match and didn't get past the end index
if (null === $index || $index > $end) {
return;
}
// initialise the result array with the current index
$result = array($index => $this[$index]);
// advance cursor to the current position
$currIdx = $index;
// iterate through the remaining tokens in the sequence
foreach ($sequence as $key => $token) {
$currIdx = $this->getNextMeaningfulToken($currIdx);
// ensure we didn't go too far
if (null === $currIdx || $currIdx > $end) {
return;
}
if (!$this[$currIdx]->equals($token, Token::isKeyCaseSensitive($caseSensitive, $key))) {
// not a match, restart the outer loop
continue 2;
}
// append index to the result array
$result[$currIdx] = $this[$currIdx];
}
// do we have a complete match?
// hint: $result is bigger than $sequence since the first token has been removed from the latter
if (count($sequence) < count($result)) {
return $result;
}
}
}
private function fixByToken(Token $token, $index)
{
foreach ($this->tokenCallbackMap as $kind => $callback) {
if (!$token->isGivenKind($kind)) {
continue;
}
$callback = $this->tokenCallbackMap[$kind];
$this->{$callback}($index);
return;
}
}
/**
* @dataProvider provideIsKeyCaseSensitive
*/
public function testIsKeyCaseSensitive($isKeyCaseSensitive, $caseSensitive, $key)
{
$this->assertSame($isKeyCaseSensitive, Token::isKeyCaseSensitive($caseSensitive, $key));
}
public function testIsEmpty()
{
$braceToken = $this->getBraceToken();
$this->assertFalse($braceToken->isEmpty());
$braceToken->setContent('');
$this->assertTrue($braceToken->isEmpty());
$whitespaceToken = new Token(array(T_WHITESPACE, ' '));
$this->assertFalse($whitespaceToken->isEmpty());
$whitespaceToken->setContent('');
$this->assertFalse($whitespaceToken->isEmpty());
$whitespaceToken->override(array(null, ''));
$this->assertTrue($whitespaceToken->isEmpty());
$whitespaceToken = new Token(array(T_WHITESPACE, ' '));
$whitespaceToken->clear();
$this->assertTrue($whitespaceToken->isEmpty());
}
private function isOperator(Token $token)
{
static $arrayOperators = array(T_AND_EQUAL => true, T_BOOLEAN_AND => true, T_BOOLEAN_OR => true, T_CONCAT_EQUAL => true, T_DIV_EQUAL => true, T_DOUBLE_ARROW => true, T_IS_EQUAL => true, T_IS_GREATER_OR_EQUAL => true, T_IS_IDENTICAL => true, T_IS_NOT_EQUAL => true, T_IS_NOT_IDENTICAL => true, T_IS_SMALLER_OR_EQUAL => true, T_LOGICAL_AND => true, T_LOGICAL_OR => true, T_LOGICAL_XOR => true, T_MINUS_EQUAL => true, T_MOD_EQUAL => true, T_MUL_EQUAL => true, T_OR_EQUAL => true, T_PLUS_EQUAL => true, T_SL => true, T_SL_EQUAL => true, T_SR => true, T_SR_EQUAL => true, T_XOR_EQUAL => true);
static $nonArrayOperators = array('=' => true);
return $token->isArray() ? isset($arrayOperators[$token->getId()]) : isset($nonArrayOperators[$token->getContent()]);
}
/**
* Transforms the heredoc start token to nowdoc notation.
*
* @param Token $token
*/
private function convertToNowdoc(Token $token)
{
$token->setContent(preg_replace('/(?<=^<<<)"?(.*?)"?$/', '\'$1\'', $token->getContent()));
}
private function fixWhitespace(Token $token)
{
if ($token->isWhitespace() && !$token->isWhitespace(" \t")) {
$token->setContent(rtrim($token->getContent()) . ' ');
}
}
private function fixIndents(Tokens $tokens)
{
$classyTokens = Token::getClassyTokenKinds();
$classyAndFunctionTokens = array_merge(array(T_FUNCTION), $classyTokens);
$controlTokens = $this->getControlTokens();
$indentTokens = array_filter(array_merge($classyAndFunctionTokens, $controlTokens), function ($item) {
return T_SWITCH !== $item;
});
$tokensAnalyzer = new TokensAnalyzer($tokens);
for ($index = 0, $limit = count($tokens); $index < $limit; ++$index) {
$token = $tokens[$index];
// if token is not a structure element - continue
if (!$token->isGivenKind($indentTokens)) {
continue;
}
// do not change indent for lambda functions
if ($token->isGivenKind(T_FUNCTION) && $tokensAnalyzer->isLambda($index)) {
continue;
}
// do not change indent for `while` in `do ... while ...`
if ($token->isGivenKind(T_WHILE) && $tokensAnalyzer->isWhilePartOfDoWhile($index)) {
continue;
}
if ($token->isGivenKind($classyAndFunctionTokens)) {
$startBraceIndex = $tokens->getNextTokenOfKind($index, array(';', '{'));
$startBraceToken = $tokens[$startBraceIndex];
} else {
$parenthesisEndIndex = $this->findParenthesisEnd($tokens, $index);
$startBraceIndex = $tokens->getNextNonWhitespace($parenthesisEndIndex);
$startBraceToken = $tokens[$startBraceIndex];
}
// structure without braces block - nothing to do, e.g. do { } while (true);
if (!$startBraceToken->equals('{')) {
continue;
}
$endBraceIndex = $tokens->findBlockEnd(Tokens::BLOCK_TYPE_CURLY_BRACE, $startBraceIndex);
$indent = $this->detectIndent($tokens, $index);
// fix indent near closing brace
$tokens->ensureWhitespaceAtIndex($endBraceIndex - 1, 1, "\n" . $indent);
// fix indent between braces
$lastCommaIndex = $tokens->getPrevTokenOfKind($endBraceIndex - 1, array(';', '}'));
$nestLevel = 1;
for ($nestIndex = $lastCommaIndex; $nestIndex >= $startBraceIndex; --$nestIndex) {
$nestToken = $tokens[$nestIndex];
if ($nestToken->equals(')')) {
$nestIndex = $tokens->findBlockEnd(Tokens::BLOCK_TYPE_PARENTHESIS_BRACE, $nestIndex, false);
continue;
}
if (1 === $nestLevel && $nestToken->equalsAny(array(';', '}'))) {
$nextNonWhitespaceNestIndex = $tokens->getNextNonWhitespace($nestIndex);
$nextNonWhitespaceNestToken = $tokens[$nextNonWhitespaceNestIndex];
if (!$nextNonWhitespaceNestToken->isComment() && !($nestToken->equals('}') && $nextNonWhitespaceNestToken->equalsAny(array(';', ',', ']', array(CT_ARRAY_SQUARE_BRACE_CLOSE)))) && !($nestToken->equals('}') && $nextNonWhitespaceNestToken->equals('(')) && !($nestToken->equals('}') && $tokens[$nestIndex - 1]->equalsAny(array('"', "'", array(T_CONSTANT_ENCAPSED_STRING))))) {
if ($nextNonWhitespaceNestToken->isGivenKind($this->getControlContinuationTokens()) || $nextNonWhitespaceNestToken->isGivenKind(T_WHILE) && $tokensAnalyzer->isWhilePartOfDoWhile($nextNonWhitespaceNestIndex)) {
$whitespace = ' ';
} else {
$nextToken = $tokens[$nestIndex + 1];
$nextWhitespace = '';
if ($nextToken->isWhitespace()) {
$nextWhitespace = rtrim($nextToken->getContent(), " \t");
if (strlen($nextWhitespace) && "\n" === $nextWhitespace[strlen($nextWhitespace) - 1]) {
$nextWhitespace = substr($nextWhitespace, 0, -1);
}
}
$whitespace = $nextWhitespace . "\n" . $indent;
if (!$nextNonWhitespaceNestToken->equals('}')) {
$whitespace .= ' ';
}
}
$tokens->ensureWhitespaceAtIndex($nestIndex + 1, 0, $whitespace);
}
}
if ($nestToken->equals('}')) {
++$nestLevel;
continue;
}
if ($nestToken->equals('{')) {
--$nestLevel;
continue;
}
}
// fix indent near opening brace
if (isset($tokens[$startBraceIndex + 2]) && $tokens[$startBraceIndex + 2]->equals('}')) {
$tokens->ensureWhitespaceAtIndex($startBraceIndex + 1, 0, "\n" . $indent);
} elseif (!$tokens[$index]->isClassy()) {
$nextToken = $tokens[$startBraceIndex + 1];
$nextNonWhitespaceToken = $tokens[$tokens->getNextNonWhitespace($startBraceIndex)];
// set indent only if it is not a case, when comment is following { in same line
if (!$nextNonWhitespaceToken->isComment() || !($nextToken->isWhitespace() && $nextToken->isWhitespace(" \t")) && substr_count($nextToken->getContent(), "\n") === 1) {
$tokens->ensureWhitespaceAtIndex($startBraceIndex + 1, 0, "\n" . $indent . ' ');
}
} else {
$tokens->ensureWhitespaceAtIndex($startBraceIndex + 1, 0, "\n" . $indent . ' ');
}
if ($token->isGivenKind($classyTokens)) {
$tokens->ensureWhitespaceAtIndex($startBraceIndex - 1, 1, "\n" . $indent);
} elseif ($token->isGivenKind(T_FUNCTION)) {
$closingParenthesisIndex = $tokens->getPrevTokenOfKind($startBraceIndex, array(')'));
$prevToken = $tokens[$closingParenthesisIndex - 1];
if ($prevToken->isWhitespace() && false !== strpos($prevToken->getContent(), "\n")) {
$tokens->ensureWhitespaceAtIndex($startBraceIndex - 1, 1, ' ');
} else {
$tokens->ensureWhitespaceAtIndex($startBraceIndex - 1, 1, "\n" . $indent);
}
} else {
$tokens->ensureWhitespaceAtIndex($startBraceIndex - 1, 1, ' ');
}
// reset loop limit due to collection change
$limit = count($tokens);
}
}
public function isCandidate(Tokens $tokens)
{
return $tokens->isAnyTokenKindsFound(Token::getClassyTokenKinds());
}
public function findSequence(array $sequence, $start = 0, $end = null, $caseSensitive = true)
{
if (null === $end) {
$end = count($this) - 1;
}
if (!count($sequence)) {
throw new \InvalidArgumentException('Invalid sequence');
}
foreach ($sequence as $key => $token) {
if (!$token instanceof Token) {
if (is_array($token) && !isset($token[1])) {
$token[1] = '';
}
$token = new Token($token);
}
if ($token->isWhitespace() || $token->isComment() || $token->isEmpty()) {
throw new \InvalidArgumentException(sprintf('Non-meaningful token at position: %s', $key));
}
}
$key = key($sequence);
$firstCs = Token::isKeyCaseSensitive($caseSensitive, $key);
$firstToken = $sequence[$key];
unset($sequence[$key]);
$index = $start - 1;
while (null !== $index && $index <= $end) {
$index = $this->getNextTokenOfKind($index, array($firstToken), $firstCs);
if (null === $index || $index > $end) {
return;
}
$result = array($index => $this[$index]);
$currIdx = $index;
foreach ($sequence as $key => $token) {
$currIdx = $this->getNextMeaningfulToken($currIdx);
if (null === $currIdx || $currIdx > $end) {
return;
}
if (!$this[$currIdx]->equals($token, Token::isKeyCaseSensitive($caseSensitive, $key))) {
continue 2;
}
$result[$currIdx] = $this[$currIdx];
}
if (count($sequence) < count($result)) {
return $result;
}
}
}
