/**
* Initializes the token map.
*
* The token map maps the PHP internal token identifiers
* to the identifiers used by the Parser. Additionally it
* maps T_OPEN_TAG_WITH_ECHO to T_ECHO and T_CLOSE_TAG to ';'.
*/
protected static function initTokenMap()
{
if (!self::$tokenMap) {
self::$tokenMap = array();
// 256 is the minimum possible token number, as everything below
// it is an ASCII value
for ($i = 256; $i < 1100; ++$i) {
// T_DOUBLE_COLON is equivalent to T_PAAMAYIM_NEKUDOTAYIM
if (T_DOUBLE_COLON === $i) {
self::$tokenMap[$i] = PHPParser_Parser::T_PAAMAYIM_NEKUDOTAYIM;
// T_OPEN_TAG_WITH_ECHO with dropped T_OPEN_TAG results in T_ECHO
} elseif (T_OPEN_TAG_WITH_ECHO === $i) {
self::$tokenMap[$i] = PHPParser_Parser::T_ECHO;
// T_CLOSE_TAG is equivalent to ';'
} elseif (T_CLOSE_TAG === $i) {
self::$tokenMap[$i] = ord(';');
// and the others can be mapped directly
} elseif ('UNKNOWN' !== ($name = snow_token_name($i)) && defined($name = 'PHPParser_Parser::' . $name)) {
self::$tokenMap[$i] = constant($name);
}
}
}
}
public function get_tokens($tmp_file)
{
$py_file = dirname(__FILE__) . "/../../python/snow/lexer/lex-to-json.py";
$json = shell_exec("python " . $py_file . " " . $tmp_file);
$python_tokens = json_decode($json, true);
if (!$python_tokens) {
var_dump($json);
die;
}
$debug = array();
$php_tokens = array(array(T_OPEN_TAG, '<?php ', 1));
foreach ($python_tokens as $t) {
$first = true;
foreach ((array) $this->translate_token($t) as $php_token) {
if ($php_token !== null) {
$php_tokens[] = $php_token;
$out_type = is_array($php_token) ? snow_token_name($php_token[0]) : 'LITERAL';
$out_value = is_array($php_token) ? $php_token[1] : $php_token;
$debug[] = array('in_type' => $first ? $t['type'] : '', 'in_value' => $first ? $t['value'] : '', 'out_type' => $out_type, 'out_value' => $out_value);
$first = false;
}
}
unset($php_token);
}
unset($t);
return array($php_tokens, $debug);
}
static function init_named_tokenmap()
{
// 256 is the minimum possible token number, as everything below
// it is an ASCII value
for ($i = 256; $i < 1100; ++$i) {
// T_DOUBLE_COLON is equivalent to T_PAAMAYIM_NEKUDOTAYIM
if (T_DOUBLE_COLON === $i) {
self::$named_tokenmap[$i] = 'T_PAAMAYIM_NEKUDOTAYIM';
// T_OPEN_TAG_WITH_ECHO with dropped T_OPEN_TAG results in T_ECHO
} elseif (T_OPEN_TAG_WITH_ECHO === $i) {
self::$named_tokenmap[$i] = PHPParser_Parser::T_ECHO;
// T_CLOSE_TAG is equivalent to ';'
} elseif (T_CLOSE_TAG === $i) {
self::$named_tokenmap[$i] = ord(';');
// and the others can be mapped directly
} elseif ('UNKNOWN' !== ($name = snow_token_name($i)) && defined($name = 'PHPParser_Parser::' . $name)) {
self::$named_tokenmap[$i] = $name;
}
}
self::$named_tokenmap = array_flip(array_map(function ($x) {
return str_replace("PHPParser_Parser::", "", $x);
}, self::$named_tokenmap));
}
