/** * Initializes the token map. * * The token map maps the PHP internal token identifiers * to the identifiers used by the Parser. Additionally it * maps T_OPEN_TAG_WITH_ECHO to T_ECHO and T_CLOSE_TAG to ';'. */ protected static function initTokenMap() { if (!self::$tokenMap) { self::$tokenMap = array(); // 256 is the minimum possible token number, as everything below // it is an ASCII value for ($i = 256; $i < 1100; ++$i) { // T_DOUBLE_COLON is equivalent to T_PAAMAYIM_NEKUDOTAYIM if (T_DOUBLE_COLON === $i) { self::$tokenMap[$i] = PHPParser_Parser::T_PAAMAYIM_NEKUDOTAYIM; // T_OPEN_TAG_WITH_ECHO with dropped T_OPEN_TAG results in T_ECHO } elseif (T_OPEN_TAG_WITH_ECHO === $i) { self::$tokenMap[$i] = PHPParser_Parser::T_ECHO; // T_CLOSE_TAG is equivalent to ';' } elseif (T_CLOSE_TAG === $i) { self::$tokenMap[$i] = ord(';'); // and the others can be mapped directly } elseif ('UNKNOWN' !== ($name = snow_token_name($i)) && defined($name = 'PHPParser_Parser::' . $name)) { self::$tokenMap[$i] = constant($name); } } } }
public function get_tokens($tmp_file) { $py_file = dirname(__FILE__) . "/../../python/snow/lexer/lex-to-json.py"; $json = shell_exec("python " . $py_file . " " . $tmp_file); $python_tokens = json_decode($json, true); if (!$python_tokens) { var_dump($json); die; } $debug = array(); $php_tokens = array(array(T_OPEN_TAG, '<?php ', 1)); foreach ($python_tokens as $t) { $first = true; foreach ((array) $this->translate_token($t) as $php_token) { if ($php_token !== null) { $php_tokens[] = $php_token; $out_type = is_array($php_token) ? snow_token_name($php_token[0]) : 'LITERAL'; $out_value = is_array($php_token) ? $php_token[1] : $php_token; $debug[] = array('in_type' => $first ? $t['type'] : '', 'in_value' => $first ? $t['value'] : '', 'out_type' => $out_type, 'out_value' => $out_value); $first = false; } } unset($php_token); } unset($t); return array($php_tokens, $debug); }
static function init_named_tokenmap() { // 256 is the minimum possible token number, as everything below // it is an ASCII value for ($i = 256; $i < 1100; ++$i) { // T_DOUBLE_COLON is equivalent to T_PAAMAYIM_NEKUDOTAYIM if (T_DOUBLE_COLON === $i) { self::$named_tokenmap[$i] = 'T_PAAMAYIM_NEKUDOTAYIM'; // T_OPEN_TAG_WITH_ECHO with dropped T_OPEN_TAG results in T_ECHO } elseif (T_OPEN_TAG_WITH_ECHO === $i) { self::$named_tokenmap[$i] = PHPParser_Parser::T_ECHO; // T_CLOSE_TAG is equivalent to ';' } elseif (T_CLOSE_TAG === $i) { self::$named_tokenmap[$i] = ord(';'); // and the others can be mapped directly } elseif ('UNKNOWN' !== ($name = snow_token_name($i)) && defined($name = 'PHPParser_Parser::' . $name)) { self::$named_tokenmap[$i] = $name; } } self::$named_tokenmap = array_flip(array_map(function ($x) { return str_replace("PHPParser_Parser::", "", $x); }, self::$named_tokenmap)); }