/**
* Tokenize source code.
*
* @param string $filename
* @param string $source
* @return File
*/
public function tokenize($filename, $source)
{
$file = new File($filename, $source);
$line = 1;
$column = 1;
$level = 0;
$namespaceFound = false;
$namespace = null;
$namespaceLevel = null;
foreach (token_get_all($source) as $token) {
if (is_array($token)) {
$type = $token[0];
$lexeme = $token[1];
} else {
$type = $token;
$lexeme = $token;
}
$token = new Token($type, $lexeme, $line, $column);
if ($token->hasNewline()) {
$line += $token->getNewLineCount();
$column = 1 + $token->getTrailingLineLength();
} else {
$column += $token->getLength();
}
// Namespace handling.
if ($type === T_NAMESPACE) {
$namespaceFound = true;
} elseif ($namespaceFound) {
if (in_array($type, array(T_STRING, T_NS_SEPARATOR))) {
$namespace .= $lexeme;
} elseif ($type === ';') {
$namespaceFound = false;
} elseif ($type === '{') {
$namespaceFound = false;
$namespaceLevel = $level;
}
} elseif ($type === '}' && $level - 1 === $namespaceLevel) {
$namespace = null;
$namespaceLevel = null;
} elseif (!$namespaceFound && $namespace !== null) {
$token->setNamespace($namespace);
}
// Block level increment.
if (in_array($type, array('(', '{', T_CURLY_OPEN, T_DOLLAR_OPEN_CURLY_BRACES))) {
$level++;
} elseif (in_array($type, array(')', '}'))) {
$level--;
}
$token->setLevel($level);
$file[] = $token;
}
return $file;
}
/**
* @covers spriebsch\PHPca\Token::__construct
* @covers spriebsch\PHPca\Token::setNamespace
* @covers spriebsch\PHPca\Token::getNamespace
*/
public function testNamespaceAccessors()
{
$t = new Token(T_OPEN_TAG, '<?php');
$t->setNamespace('Foo\\Bar');
$this->assertEquals('Foo\\Bar', $t->getNamespace());
}
/**
* Tokenize a file
*
* @param string $fileName the file name
* @param string $sourceCode the source code
* @return File
*/
public static function tokenize($fileName, $sourceCode)
{
Constants::init();
$class = '';
$classFound = false;
$waitForClassBegin = false;
$classCurlyLevel = 0;
$interface = '';
$interfaceFound = false;
$waitForInterfaceBegin = false;
$interfaceCurlyLevel = 0;
$function = '';
$functionFound = false;
$waitForFunctionBegin = false;
$functionCurlyLevel = 0;
$namespace = '\\';
$newNamespace = '';
$namespaceFound = false;
$namespaceStarted = false;
$level = 0;
$line = 1;
$column = 1;
$file = new File($fileName, $sourceCode);
foreach (token_get_all($sourceCode) as $token) {
if (is_array($token)) {
$id = $token[0];
$text = $token[1];
$line = $token[2];
} else {
try {
// it's not a PHP token, so we use one we have defined
$id = Constants::getTokenId($token);
$text = $token;
} catch (UnkownTokenException $e) {
throw new TokenizerException('Unknown token ' . $e->getTokenName() . ' in file ' . $fileName);
}
// This exception is not testable, because we _have_ defined all
// tokens, hopefully. It's just a safeguard to provide a decent
// error message should we ever encounter an undefined token.
// @codeCoverageIgnoreEnd
}
$tokenObj = new Token($id, $text, $line, $column);
if ($tokenObj->hasNewline()) {
// a newline resets the column count
$line += $tokenObj->getNewLineCount();
$column = 1 + $tokenObj->getTrailingWhitespaceCount();
} else {
$column += $tokenObj->getLength();
}
// We have encountered a T_NAMESPACE token before (this is indicated
// by $namespaceFound being true, so the T_STRING contains the class
// name (there will be T_WHITESPACE between T_NAMESPACE and T_STRING).
// We remember the namespace name, but do not set it until we have
// encountered the next opening brace or semicolon. We set
// $waitForNamespaceBegin to true so that we can wait for one of these.
if ($namespaceFound && $tokenObj->getId() == T_STRING || $namespaceFound && $tokenObj->getId() == T_NS_SEPARATOR) {
$newNamespace .= $tokenObj->getText();
}
// We have encountered a T_CLASS token before (this is indicated
// by $classFound being true, so the T_STRING contains the class
// name (there will be T_WHITESPACE between T_CLASS and T_STRING).
// We remember the class name, but do not set it until we have
// encountered the next opening brace. We set $waitForClassBegin
// to true so that we can wait for the next opening curly brace.
if ($classFound && $tokenObj->getId() == T_STRING) {
$class = $tokenObj->getText();
$waitForClassBegin = true;
$classFound = false;
}
// We have encountered a T_INTERFACE token before (this is indicated
// by $interfaceFound being true, so the T_STRING contains the class
// name (there will be T_WHITESPACE between T_INTERFACE and T_STRING).
// We remember the interface name, but do not set it until we have
// encountered the next opening brace. We set $waitForInterfaceBegin
// to true so that we can wait for the next opening curly brace.
if ($interfaceFound && $tokenObj->getId() == T_STRING) {
$interface = $tokenObj->getText();
$waitForInterfaceBegin = true;
$interfaceFound = false;
}
// We have encountered a T_FUNCTION token before (this is indicated
// by $functionFound being true, so the T_STRING contains the class
// name (there will be T_WHITESPACE between T_FUNCTION and T_STRING).
// We remember the function name, but do not set it until we have
// encountered the next opening brace. We set $waitForFunctionBegin
// to true so that we can wait for the next opening curly brace.
if ($functionFound && $tokenObj->getId() == T_STRING) {
$function = $tokenObj->getText();
$waitForFunctionBegin = true;
$functionFound = false;
}
// T_NAMESPACE token starts a namespace. We set $namespaceFound
// to true so that we can watch out for the namespace name (see above).
if ($tokenObj->getId() == T_NAMESPACE) {
// Reset the current namespace. It is a PHPCa convention to
// always make namespace statements themselves part of the
// global namespace.
$namespace = '\\';
$namespaceFound = true;
}
// If we encounter a T_CLASS token, we have found a class definition.
// We set $classFound to true so that we can watch out for the class
// name (see above).
if ($tokenObj->getId() == T_CLASS) {
$classFound = true;
}
// If we encounter a T_INTERFACE token, we have found an interface definition.
// We set $interfaceFound to true so that we can watch out for the interface
// name (see above).
if ($tokenObj->getId() == T_INTERFACE) {
$interfaceFound = true;
}
// If we encounter a T_FUNCTION token, we have found a function.
// We set $functionFound to true so that we can watch out for the
// function name (see above).
if ($tokenObj->getId() == T_FUNCTION) {
$functionFound = true;
}
// A semicolon can end the namespace declaration. If we encounter
// a semicolon in $namespaceFound mode, we switch the mode which
// indicates that the full namespace name has been parsed.
if ($namespaceFound && $tokenObj->getId() == T_SEMICOLON) {
$namespaceStarted = true;
$namespaceFound = false;
}
$id = $tokenObj->getId();
if ($id == T_SEMICOLON) {
if ($waitForFunctionBegin) {
$functionCurlyLevel = $level;
$waitForFunctionBegin = false;
}
}
// Opening curly brace opens another block, thus increases the level.
if ($id == T_OPEN_CURLY || $id == T_CURLY_OPEN || $id == T_DOLLAR_OPEN_CURLY_BRACES) {
$level++;
// An opening curly brace can end the namespace declaration.
// If we encounter one while in $namespaceFound mode, we switch
// the mode which indicates that the full namespace name has
// been parsed.
if ($namespaceFound) {
$namespace = $newNamespace;
$newNamespace = '';
$namespaceFound = false;
}
// If we encounter the opening curly brace of a class (this happens
// when $waitForClassBegin is true), we remember the block level of
// this brace so that we can end the class when we encounter the
// matching closing tag.
if ($waitForClassBegin) {
$classCurlyLevel = $level;
$waitForClassBegin = false;
}
// If we encounter the opening curly brace of an interface (this happens
// when $waitForInterfaceBegin is true), we remember the block level of
// this brace so that we can end the interface when we encounter the
// matching closing tag.
if ($waitForInterfaceBegin) {
$interfaceCurlyLevel = $level;
$waitForInterfaceBegin = false;
}
// If we encounter the opening curly brace of a class (this happens
// when $waitForClassBegin is true), we remember the block level of
// this brace so that we can end the class when we encounter the
// matching closing tag.
if ($waitForFunctionBegin) {
$functionCurlyLevel = $level;
$waitForFunctionBegin = false;
}
}
// Since we assemble any new namespace name in $newNamespace,
// we can safely always set the $namespace.
$tokenObj->setNamespace($namespace);
if ($namespaceStarted) {
$namespaceStarted = false;
$namespace = $newNamespace;
$newNamespace = '';
}
// This also sets the class when we are outside the class,
// which is harmless because we then just set an emtpy string.
if (!$waitForClassBegin) {
if (substr($class, 0, 1) == '\\' || $namespace == '\\') {
$classname = $class;
} else {
if ($class != '') {
$classname = $namespace . '\\' . $class;
} else {
$classname = '';
}
}
$tokenObj->setClass($classname);
}
// This also sets the interface when we are outside the interface,
// which is harmless because we then just set an emtpy string.
if (!$waitForInterfaceBegin) {
if (substr($interface, 0, 1) == '\\' || $namespace == '\\') {
$interfaceName = $interface;
} else {
if ($interface != '') {
$interfaceName = $namespace . '\\' . $interface;
} else {
$interfaceName = '';
}
}
$tokenObj->setInterface($interfaceName);
}
// This also sets the function when we are outside the function,
// which is harmless because we then just set an emtpy string.
if (!$waitForFunctionBegin) {
$tokenObj->setFunction($function);
}
$tokenObj->setBlockLevel($level);
// Closing curly decreases the block level. We do this *after*
// we have set the block leven in the current token, so that
// the closing curly's level matches the level of its opening brace.
if ($tokenObj->getId() == T_CLOSE_CURLY) {
$level--;
// We get away with not dealing with namespace ends, since
// non-namespaced code is not allowed when there is at least
// one namespace in a file. So any namespace either implicitly
// ends at the end of a file, or another namespace starts,
// implicitly "ending" the previous namespace.
// If we are inside a class and the closing brace matches the
// opening brace of that class, the block/class has ended.
if ($class != '' && $tokenObj->getBlockLevel() == $classCurlyLevel) {
$class = '';
$classCurlyLevel = 0;
}
// If we are inside an interface and the closing brace matches the
// opening brace of that interface, the block/interface has ended.
if ($interface != '' && $tokenObj->getBlockLevel() == $interfaceCurlyLevel) {
$interface = '';
$interfaceCurlyLevel = 0;
}
// If we are inside a function and the closing brace matches the
// opening brace of that function, the block/function has ended.
if ($function != '' && $tokenObj->getBlockLevel() == $functionCurlyLevel) {
$function = '';
$functionCurlyLevel = 0;
}
}
$file->add($tokenObj);
}
return $file;
}
