private function _runPHP() { $this->_source = "return " . $this->_source . ";"; if (function_exists("token_get_all")) { //tokenizer extension may be disabled $php = "<?php\n" . $this->_source . "\n?>"; $tokens = token_get_all($php); foreach ($tokens as $token) { $type = $token[0]; if (is_long($type)) { if (in_array($type, array(T_OPEN_TAG, T_RETURN, T_WHITESPACE, T_ARRAY, T_LNUMBER, T_DNUMBER, T_CONSTANT_ENCAPSED_STRING, T_DOUBLE_ARROW, T_CLOSE_TAG, T_NEW))) { continue; } if ($type == T_STRING) { $func = strtolower($token[1]); if (in_array($func, array("mongoid", "mongocode", "mongodate", "mongoregex", "mongobindata", "mongoint32", "mongoint64", "mongodbref", "mongominkey", "mongomaxkey", "mongotimestamp", "true", "false", "null"))) { continue; } } exit("For your security, we stoped data parsing at '(" . token_name($type) . ") " . $token[1] . "'."); } } } return eval($this->_source); }
/** * Splits the provided `$code` into PHP language tokens. * * @param string $code Source code to be tokenized. * @param array $options Options consists of: * -'wrap': Boolean indicating whether or not to wrap the supplied * code in PHP tags. * -'ignore': An array containing PHP language tokens to ignore. * -'include': If supplied, an array of the only language tokens * to include in the output. * @return array An array of tokens in the supplied source code. */ public static function tokenize($code, array $options = array()) { $defaults = array('wrap' => true, 'ignore' => array(), 'include' => array()); $options += $defaults; $tokens = array(); $line = 1; if ($options['wrap']) { $code = "<?php {$code}?>"; } foreach (token_get_all($code) as $token) { $token = isset($token[1]) ? $token : array(null, $token, $line); list($id, $content, $line) = $token; $name = $id ? token_name($id) : $content; if (!empty($options['include'])) { if (!in_array($name, $options['include']) && !in_array($id, $options['include'])) { continue; } } if (!empty($options['ignore'])) { if (in_array($name, $options['ignore']) || in_array($id, $options['ignore'])) { continue; } } $tokens[] = array('id' => $id, 'name' => $name, 'content' => $content, 'line' => $line); if ($id === T_WHITESPACE) { $line += count(preg_split('/\\r\\n|\\r|\\n/', $content)) - 1; } } if ($options['wrap'] && empty($options['include'])) { $tokens = array_slice($tokens, 1, count($tokens) - 2); } return $tokens; }
function token_name_nl($token) { if ($token === T_NEW_LINE) { return 'T_NEW_LINE'; } return token_name($token); }
/** * Output user friendly error messages * * The message is based on the token ID * * @param array $errors List of errors for the file * @param string $file Path to the file that was tokenized * @return void */ protected function _outputFriendlyErrors($errors, $file) { $file = $this->_normalizeFilePath($file); $this->log(sprintf('%s (%d errors)', $file, sizeof($errors)), 'info'); foreach ($errors as $error) { switch ($error[0]) { case T_EXIT: $this->_outputError('die() statements not allowed in views', $error); break; case T_ECHO: $this->_outputError('echo() statements not allowed in views', $error); break; case T_VARIABLE: $this->_outputError('Unsafe variable output for "' . $error[1] . '". Please wrap in h()', $error); break; case T_REQUIRE: $this->_outputError('Using require() inside a View is not allowed, use $this->element()', $error); break; case T_INCLUDE: $this->_outputError('Using include() inside a View is not allowed, use $this->element()', $error); break; default: debug(token_name($error[0])); debug($error); break; } } }
/** * Разбор блока * * @param array $token * @param integer $i * @param integer $init * @return string */ public static function analizeToken($token, $i, $init) { $result = ""; switch (token_name((int) $token[0])) { case "T_WHITESPACE": case "T_ENCAPSED_AND_WHITESPACE": // New line between commands fixer $result .= " "; break; // [ FIXME ]: Implement a fix for this situation // [ FIXME ]: Implement a fix for this situation case "T_CONSTANT_ENCAPSED_STRING": // New line in string definition fixer $result .= $token[1]; break; case "T_OPEN_TAG": if ($i == $init && is_array($token[1])) { // Last weird behavior of PHP Tokenizer... it puts // the first PHP command as part of the T_OPEN_TAG $result .= Miaox_Aop_CodeCruncher::analizeToken($token[1], $i, $init); } else { $result .= trim($token[1]); } break; case "T_COMMENT": case "T_ML_COMMENT": case "T_DOC_COMMENT": // Do nothing break; default: $result .= $token[1]; break; } return $result; }
public function getAllUseStatements($contents) { $tokens = token_get_all($contents); $shortName = $className = ''; $useNamespace = $useFlag = $asFlag = 0; $result = array(); foreach ($tokens as $token) { if (is_array($token)) { if (token_name($token[0]) == 'T_WHITESPACE') { // $useNamespace = 0; continue; } if (token_name($token[0]) == 'T_NAMESPACE') { $useNamespace = 1; continue; } if (token_name($token[0]) == 'T_CLASS') { break; } if (token_name($token[0]) == 'T_USE') { $useFlag = 1; $useNamespace = 0; continue; } if (strtolower($token[1]) == 'as') { $asFlag = 1; continue; } if ($useNamespace) { $this->currentNamespace .= $token[1]; } if ($useFlag && !$asFlag) { $className .= $token[1]; } if ($asFlag) { $shortName = $token[1]; $asFlag = 0; } // start keeping class } else { if ($useFlag && ($token == ';' || $token == ',')) { $className = trim($className); if (!$shortName) { try { $shortName = $this->getShortName($className); } catch (\Exception $e) { // Errored continue; } } $result[$shortName] = $className; if ($token == ',') { $useFlag = 1; } $shortName = $className = ''; } } } return $result; }
public function mutate(MutationInterface $original, $index) { $token = $original->getTokens()->offsetGet($index); if ($token->getType() !== T_ELSE) { throw new \UnexpectedValueException(sprintf('invalid token "%s" given in %s', token_name($token->getType()), get_class($this))); } // look for the closing bracket $tokens = $original->getTokens(); $len = $tokens->count(); $end = false; for ($i = $index; $i < $len; $i++) { $token = $tokens->offsetGet($i); if ($token->getType() === T_STRING && $token->getValue() === '}') { $end = $i; break; } } if (false === $end) { throw new \OutOfRangeException('closing bracket not found for else'); } // remove all concerned tokens $tokens = $tokens->remove($index, $end); $new = new \Hal\MutaTesting\Mutation\Mutation(); $new->setTokens($tokens)->setUnit($original->getUnit())->setSourceFile($original->getSourceFile())->setTestFile($original->getTestFile())->setMutedTokensIndexes(array_merge($original->getMutedTokensIndexes(), range($index, $end))); return $new; }
public static function tokenize($code, $options = array()) { $defaults = array('wrap' => true, 'ignore' => array(), 'include' => array()); $options += $defaults; $tokens = array(); $line = 1; if ($options['wrap']) { $code = "<?php {$code}?>"; } foreach (token_get_all($code) as $token) { $token = is_array($token) ? $token : array(null, $token, $line); list($id, $content, $line) = $token; $name = $id ? token_name($id) : $content; if (!empty($options['include'])) { if (!in_array($name, $options['include']) && !in_array($id, $options['include'])) { continue; } } if (in_array($name, $options['ignore']) || in_array($id, $options['ignore'])) { continue; } $tokens[] = compact('id', 'name', 'content', 'line'); } if ($options['wrap'] && empty($options['include'])) { $tokens = array_slice($tokens, 1, count($tokens) - 2); } return $tokens; }
/** * @return string */ public function getName() { if ('UNKNOWN' === $this->name) { $this->setName(token_name($this->getId())); } return $this->name; }
private function getFunction($idTocken) { $function = $this->tokens[$idTocken][1]; $level = 0; $nom = null; $canBeStore = false; for ($idTocken = $idTocken + 1; $idTocken < count($this->tokens); $idTocken++) { $token = $this->tokens[$idTocken]; if (is_null($nom) && token_name($token[0]) == 'T_STRING') { $nom = $token[1]; } if ($token === '{') { $level++; } elseif (is_array($token) && $token[0] == 383) { $level++; } elseif ($token === '}') { $level--; $canBeStore = true; } if (is_array($token)) { $function .= $token[1]; } else { $function .= $token; } if ($canBeStore && $level == 0) { self::$history[$this->fileName][$nom] = $function; return $idTocken; } } }
public function getClassnameForToken(Token $token) { $type = $token->getType(); $value = $token->getValue(); $classname = null; switch ($type) { case T_STRING: // case of operators if (isset(self::$OPERATOR_MAP[$value])) { $classname = self::$OPERATOR_MAP[$value]; } break; default: $classname = token_name($type); break; } // camelcase $classname = strtolower($classname); $classname = preg_replace_callback('/_(.?)/', function ($matches) { return strtoupper($matches[1]); }, $classname); $classname = preg_replace('!(^t)!', '', $classname); if (null !== $classname) { $classname = '\\Hal\\MutaTesting\\Mutater\\Mutater' . $classname; } return $classname; }
public function __construct($tokens) { // The stack, each entry should look like: // [ $func, $index $stack = []; $history = []; // Need a first sweep to detect all possible entry points into the code $entrypoints = []; foreach ($tokens as $tok) { // Find all "function" tokens and the first code lines after opening tags } // Scann all the tokens here, try to follow execution flow building a stack and matching it against criteria foreach ($tokens as $tok) { switch ($tok[0]) { case T_WHITESPACE: break; default: if (is_string($tok[0])) { printf("Skipping stringtoken '%s'\n", $tok[0]); } else { printf("Skipping token %s (%s)\n", token_name($tok[0]), "'" . join("','", array_slice($tok, 1)) . "'"); } break; } } }
protected function execute(InputInterface $in, OutputInterface $out) { $file = $in->getArgument('target'); if (!is_file($file) || !is_readable($file)) { $out->writeln(sprintf('<error>"%s" is not a valid file!</error>', $file)); return; } $tokenizer = $in->getOption('tokenizer'); if ($tokenizer === 'php') { $tokens = token_get_all(file_get_contents($file)); foreach ($tokens as $token) { if (is_array($token)) { $out->writeln(sprintf('%s(%s) "%s"', token_name($token[0]), $token[0], $token[1])); } else { $out->writeln('"' . $token . '"'); } } } elseif ($tokenizer === 'lib') { $lexer = ExtendedEmulativeLexer::createDefaultInstance(); $lexer->startLexing(file_get_contents($file)); $offset = 0; while ($id = $lexer->getNextToken()) { if (is_string($name = $this->libTokenToTokenName($id))) { $out->writeln($name . '(' . $lexer->getTokens()[$offset][0] . ') ' . $lexer->getTokens()[$offset][1]); } else { $out->writeln('Discarded token with id ' . $id); } $offset++; } } else { $out->writeln(sprintf('<error>"%s" is not a valid value for the tokenizer option</error>', $tokenizer)); } }
function getType() { if (is_string($this->data)) { switch ($this->data) { case T_BLOCK_OPEN: return "T_BLOCK_OPEN"; case T_BLOCK_CLOSE: return "T_BLOCK_CLOSE"; case T_ARRAY_INDEX_OPEN: return "T_ARRAY_INDEX_OPEN"; case T_ARRAY_INDEX_CLOSE: return "T_ARRAY_INDEX_CLOSE"; case T_ROUND_BRACE_OPEN: return "T_ROUND_BRACE_OPEN"; case T_ROUND_BRACE_CLOSE: return "T_ROUND_BRACE_CLOSE"; case T_EQUAL: return "T_EQUAL"; case T_COLON: return "T_COLON"; case T_SEMICOLON: return "T_SEMICOLON"; default: return "T_STRING"; } } else { list($id, $text) = $this->data; return token_name($id); } }
function getTokenType($num) { if ($num > count($this->tokens) || $num < 0) { throw new InvalidParameterException("Invalid token number!!"); } $tk = $this->tokens[$num]; if (is_string($tk)) { switch ($tk) { case '{': return "T_BLOCK_OPEN"; case '}': return "T_BLOCK_CLOSE"; case '[': return "T_ARRAY_INDEX_OPEN"; case ']': return "T_ARRAY_INDEX_CLOSE"; case '(': return "T_ROUND_BRACE_OPEN"; case ')': return "T_ROUND_BRACE_CLOSE"; case '=': return "T_EQUAL"; case ',': return "T_COLON"; case ';': return "T_SEMICOLON"; default: return "T_STRING"; } } else { list($id, $text) = $tk; return token_name($id); } }
/** * @return string */ public function getName() { if (!$this->type) { return 'F_NONSTANDARD_TOKEN'; } return token_name($this->type); }
public function testParsingAccessLevels() { $statics = $this->parseSelf()->getStatics(); $levels = array('nolevel' => null, 'public' => T_PUBLIC, 'public2' => T_PUBLIC, 'protected' => T_PROTECTED, 'protected2' => T_PROTECTED, 'private' => T_PRIVATE, 'private2' => T_PRIVATE, 'nolevel_after_private' => null); foreach ($levels as $var => $level) { $this->assertEquals($level, $statics[__CLASS__][$var]['access'], 'Variable ' . $var . ' has ' . ($level ? token_name($level) : 'no') . ' access level'); } }
/** * Return the token name as a string. */ public static function getTokenName($token) { if (is_int($token[self::T_KEY])) { return token_name($token[self::T_KEY]); } else { return $token[self::T_KEY]; } }
public function getName() : string { if (!is_string($this->id)) { return token_name($this->id); } else { return $this->id; } }
/** * @covers Veles\Tools\PhpToken::getName */ public function testGetName() { $this->object->setId(334); $expected = token_name(334); $result = $this->object->getName(); $msg = 'Wrong behavior of PhpToken::getName()'; $this->assertSame($expected, $result, $msg); }
/** * Returns the token name. * * @param int $type * @return string */ public static function getName($type) { if ($type == T_PUNCTUATION) { $type = 'T_PUNCTUATION'; } else { $type = token_name($type); } return $type; }
/** * Turns the operator constant into a human readable string * @param integer $code - operator ID code * @see http://php.net/manual/en/tokens.php * @return string */ public function getOperatorName($code) { $code = (int) $code; $str = token_name($code); if (($r = strtolower($str)) == 'unknown') { return $r; } return isset($this->triggerNameCache[$code]) ? $this->triggerNameCache[$code] : ($this->triggerNameCache[$code] = substr(strtolower($str), 2, strlen($str))); }
function prettyprint_tokens($tokens) { foreach ($tokens as $t) { if (is_array($t)) { echo token_name($t[0]) === 'T_WHITESPACE' ? "" : token_name($t[0]) . ': ' . trim($t[1]) . "\n"; } else { echo $t . "\n"; } } }
/** * @param int $tokenId * * @return string */ public static function tokenName($tokenId) { $tokenName = token_name($tokenId); if ('UNKNOWN' === $tokenName) { if (array_key_exists($tokenId, self::$tokenNames)) { return self::$tokenNames[$tokenId]; } } return $tokenName; }
/** * Will iterate over each line checking if any weak comparison operators * are used within the code. * * @param Testable $testable The testable object * @return void */ public function apply($testable, array $config = array()) { $tokens = $testable->tokens(); $message = 'Weak comparison operator {:key} used, try {:value} instead'; $filtered = $testable->findAll(array_keys($this->inspectableTokens)); foreach ($filtered as $id) { $token = $tokens[$id]; $this->addWarning(array('message' => String::insert($message, array('key' => token_name($token['id']), 'value' => $this->inspectableTokens[$token['id']])), 'line' => $token['line'])); } }
protected function mutateOne(MutationInterface $original, $index, $expected, Token $newToken) { $token = $original->getTokens()->offsetGet($index); if ($token->getType() !== $expected) { throw new \UnexpectedValueException(sprintf('invalid token "%s" given in %s', token_name($token->getType()), get_class($this))); } $new = new \Hal\MutaTesting\Mutation\Mutation(); $new->setTokens($original->getTokens()->replace($index, $newToken))->setUnit($original->getUnit())->setSourceFile($original->getSourceFile())->setTestFile($original->getTestFile())->setMutedTokensIndexes(array_merge($original->getMutedTokensIndexes(), array($index))); return $new; }
/** * Return the name of a token, including the NEW_LINE one. * * @return String the name of the token */ public function getName() { $tagNames = array(T_NEW_LINE => 'T_NEW_LINE', T_TAB => 'T_TAB', T_SEMICOLON => 'T_SEMICOLON', T_BRACES_OPEN => 'T_BRACES_OPEN', T_BRACES_CLOSE => 'T_BRACES_CLOSE', T_PARENTHESIS_OPEN => 'T_PARENTHESIS_OPEN', T_PARENTHESIS_CLOSE => 'T_PARENTHESIS_CLOSE', T_COMMA => 'T_COMMA', T_EQUAL => 'T_EQUAL', T_CONCAT => 'T_CONCAT', T_COLON => 'T_COLON', T_MINUS => 'T_MINUS', T_PLUS => 'T_PLUS', T_IS_GREATER => 'T_IS_GREATER', T_IS_SMALLER => 'T_IS_SMALLER', T_MULTIPLY => 'T_MULTIPLY', T_DIVIDE => 'T_DIVIDE', T_QUESTION_MARK => 'T_QUESTION_MARK', T_MODULO => 'T_MODULO', T_EXCLAMATION_MARK => 'T_EXCLAMATION_MARK', T_AMPERSAND => 'T_AMPERSAND', T_SQUARE_BRACKET_OPEN => 'T_SQUARE_BRACKET_OPEN', T_SQUARE_BRACKET_CLOSE => 'T_SQUARE_BRACKET_CLOSE', T_AROBAS => 'T_AROBAS', T_UNKNOWN => 'T_UNKNOWN'); if (isset($tagNames[$this->id])) { $result = $tagNames[$this->id]; } else { $result = token_name($this->id); } return $result; }
public function __construct($container) { $this->container = $container; for ($i = 100; $i < 500; $i++) { if (($name = @token_name($i)) == 'UNKNOWN') { continue; } $this->tokens[$i] = $name; } }
/** * Constructs a token object. * * @param string $code * @param int|null $value * @param int|null $line * * @throws \InvalidArgumentException */ public function __construct($code, $value = null, $line = null) { if (is_array($code)) { list($value, $code, $line) = array_pad($code, 3, null); } $this->code = $code; $this->value = $value; $this->line = $line; $this->name = $value ? token_name($value) : null; }
function f($code) { $tokens = token_get_all($code); foreach ($tokens as $token) { $id = is_string($token) ? "'{$token}'" : token_name($token[0]); $t = is_string($token) ? $token : $token[1]; echo "{$id} => "; var_dump(addcslashes($t, "\r\n")); } }