/** * @dataProvider provideTestHaltCompiler */ public function testHandleHaltCompiler($code, $remaining) { $this->lexer->startLexing($code); while (PHPParser_Parser::T_HALT_COMPILER !== $this->lexer->getNextToken()) { } $this->assertEquals($this->lexer->handleHaltCompiler(), $remaining); $this->assertEquals(0, $this->lexer->getNextToken()); }
/** * @dataProvider provideTestHaltCompiler */ public function testHandleHaltCompiler($code, $remaining) { $lexer = new PHPParser_Lexer($code); while (PHPParser_Parser::T_HALT_COMPILER !== $lexer->lex()) { } $this->assertEquals($lexer->handleHaltCompiler(), $remaining); $this->assertEquals(0, $lexer->lex()); }
/** * Retrieves the next token and determines the associated attributes and * returns the token id. * * @param string $value * @param string[] $startAttributes * @param string[] $endAttributes * * @return int */ public function getNextToken(&$value = null, &$startAttributes = null, &$endAttributes = null) { $tokenId = parent::getNextToken($value, $startAttributes, $endAttributes); if ($this->isTokenScalar($tokenId)) { // store original value because the value itself will be interpreted // by PHP_Parser and we want the unformatted value $endAttributes['originalValue'] = $value; } return $tokenId; }
function __construct($code) { $tmp_file = "/tmp/.snowcode"; file_put_contents($tmp_file, $code); parent::__construct(""); #var_dump(Snowscript_Lexer::$named_tokenmap, # Snowscript_Lexer::$tokenMap); die; list($this->tokens, $this->debug) = $this->get_tokens($tmp_file); unlink($tmp_file); }
/** * Initializes the token map. * * The token map maps the PHP internal token identifiers * to the identifiers used by the Parser. Additionally it * maps T_OPEN_TAG_WITH_ECHO to T_ECHO and T_CLOSE_TAG to ';'. */ protected static function initTokenMap() { if (!self::$tokenMap) { self::$tokenMap = array(); // 256 is the minimum possible token number, as everything below // it is an ASCII value for ($i = 256; $i < 1000; ++$i) { // T_DOUBLE_COLON is equivalent to T_PAAMAYIM_NEKUDOTAYIM if (T_DOUBLE_COLON === $i) { self::$tokenMap[$i] = PHPParser_Parser::T_PAAMAYIM_NEKUDOTAYIM; // T_OPEN_TAG_WITH_ECHO with dropped T_OPEN_TAG results in T_ECHO } elseif (T_OPEN_TAG_WITH_ECHO === $i) { self::$tokenMap[$i] = PHPParser_Parser::T_ECHO; // T_CLOSE_TAG is equivalent to ';' } elseif (T_CLOSE_TAG === $i) { self::$tokenMap[$i] = ord(';'); // and the others can be mapped directly } elseif ('UNKNOWN' !== ($name = token_name($i)) && defined($name = 'PHPParser_Parser::' . $name)) { self::$tokenMap[$i] = constant($name); } } } }
public function lex(&$value = null, &$line = null, &$docComment = null) { $token = parent::lex($value, $line, $docComment); // replace new keywords by their respective tokens. This is not done // if we currently are in an object access (e.g. in $obj->namespace // "namespace" stays a T_STRING tokens and isn't converted to T_NAMESPACE) if (PHPParser_Parser::T_STRING === $token && !$this->inObjectAccess) { if (isset(self::$keywords[strtolower($value)])) { return self::$keywords[strtolower($value)]; } // backslashes are replaced by T_NS_SEPARATOR tokens } elseif (92 === $token) { // ord('\\') return PHPParser_Parser::T_NS_SEPARATOR; // keep track of whether we currently are in an object access (after ->) } elseif (PHPParser_Parser::T_OBJECT_OPERATOR === $token) { $this->inObjectAccess = true; } else { $this->inObjectAccess = false; } return $token; }
public function __construct($code) { $this->tokens = array(); $this->debug = array(); $this->named_tokenmap = get_named_tokenmap(); $this->transform_token_value = array('T_VARIABLE' => '$%s'); $this->literal_tokens = array('T_PLUS' => 1, 'T_GREATER' => 1, 'T_LPAR' => 1, 'T_RPAR' => 1, 'T_MINUS' => 1, 'T_STAR' => 1, 'T_SLASH' => 1, 'T_EQUAL' => 1, 'T_AMPER' => 1, 'T_COMMA' => 1, 'T_LSQB' => 1, 'T_RSQB' => 1, 'T_QUESTION_MARK' => 1, 'T_COLON' => 1); $this->translated_tokens = array('T_NEWLINE' => ';', 'T_INDENT' => '{', 'T_DEDENT' => '}', 'T_BAND' => '&', 'T_BXOR' => '^', 'T_PERCENT' => '.', 'T_MOD' => '%', 'T_BNOT' => '~', 'T_BOR' => '|', 'T_LBRACE' => '{', 'T_RBRACE' => '}', 'T_LESS' => '<', 'T_NOT' => '!'); $this->ignored_tokens = array('T_ENDMARKER' => 1, 'T_PASS' => 1); $this->token_types_map = array('T_NAME' => 'T_VARIABLE', 'T_PHP_STRING' => 'T_STRING', 'T_BLEFT' => 'T_SL', 'T_BRIGHT' => 'T_SR', 'T_FN' => 'T_FUNCTION', 'T_DOUBLE_DOT' => 'T_PAAMAYIM_NEKUDOTAYIM', 'T_CALLABLE' => 'T_STRING', 'T_TRUE' => 'T_STRING', 'T_FALSE' => 'T_STRING', 'T_ELIF' => 'T_ELSEIF', 'T_ISA' => 'T_INSTANCEOF', 'T_DIE' => 'T_EXIT', 'T_OR' => 'T_BOOLEAN_OR', 'T_XOR' => 'T_LOGICAL_XOR', 'T_AND' => 'T_BOOLEAN_AND', 'T__OR_' => 'T_LOGICAL_OR', 'T__AND_' => 'T_LOGICAL_AND', 'T_DOT' => 'T_OBJECT_OPERATOR', 'T_NULL' => 'T_STRING', 'T_CONSTANT_NAME' => 'T_STRING', 'T_CLASS_NAME' => 'T_STRING', 'T_FLOAT_CAST' => 'T_DOUBLE_CAST', 'T_STRINGTYPE_CAST' => 'T_STRING_CAST', 'T_NEXT' => 'T_CONTINUE', 'T_PARENT' => 'T_STRING', 'T_COLON' => 'T_DOUBLE_ARROW'); $this->token_callback = array('T_STRING_WITH_CONCAT' => 1, 'T_NUMBER' => 1); $tmp_file = "/tmp/.snowcode"; file_put_contents($tmp_file, $code); parent::__construct(""); list($this->tokens, $this->debug) = $this->get_tokens($tmp_file); unlink($tmp_file); }
/** * Parses PHP code into a node tree. * * @param PHPParser_Lexer $lexer A lexer * * @return array Array of statements */ public function parse(PHPParser_Lexer $lexer) { $this->lexer = $lexer; $this->yysp = 0; // Stack pos $yysstk = array($yystate = 0); // State stack $this->yyastk = array(); // AST stack (?) $yylstk = array($yyline = 1); // Line stack $yydstk = array($yyDC = null); // Doc comment stack $yychar = -1; for (;;) { if (self::$yybase[$yystate] == 0) { $yyn = self::$yydefault[$yystate]; } else { if ($yychar < 0) { if (($yychar = $lexer->lex($yylval, $yyline, $yyDC)) < 0) { $yychar = 0; } $yychar = $yychar < self::YYMAXLEX ? self::$yytranslate[$yychar] : self::YYBADCH; $yylstk[$this->yysp] = $yyline; $yydstk[$this->yysp] = $yyDC; } if ((($yyn = self::$yybase[$yystate] + $yychar) >= 0 && $yyn < self::YYLAST && self::$yycheck[$yyn] == $yychar || $yystate < self::YY2TBLSTATE && ($yyn = self::$yybase[$yystate + self::YYNLSTATES] + $yychar) >= 0 && $yyn < self::YYLAST && self::$yycheck[$yyn] == $yychar) && ($yyn = self::$yyaction[$yyn]) != self::YYDEFAULT) { /* * >= YYNLSTATE: shift and reduce * > 0: shift * = 0: accept * < 0: reduce * = -YYUNEXPECTED: error */ if ($yyn > 0) { /* shift */ ++$this->yysp; $yysstk[$this->yysp] = $yystate = $yyn; $this->yyastk[$this->yysp] = $yylval; $yylstk[$this->yysp] = $yyline; $yydstk[$this->yysp] = $yyDC; $yychar = -1; if ($yyn < self::YYNLSTATES) { continue; } /* $yyn >= YYNLSTATES means shift-and-reduce */ $yyn -= self::YYNLSTATES; } else { $yyn = -$yyn; } } else { $yyn = self::$yydefault[$yystate]; } } for (;;) { /* reduce/error */ if ($yyn == 0) { /* accept */ return $this->yyval; } elseif ($yyn != self::YYUNEXPECTED) { /* reduce */ try { $this->{'yyn' . $yyn}($yylstk[$this->yysp - self::$yylen[$yyn]], $yydstk[$this->yysp - self::$yylen[$yyn]]); } catch (PHPParser_Error $e) { if (-1 === $e->getRawLine()) { $e->setRawLine($yyline); } throw $e; } /* Goto - shift nonterminal */ $this->yysp -= self::$yylen[$yyn]; $yyn = self::$yylhs[$yyn]; if (($yyp = self::$yygbase[$yyn] + $yysstk[$this->yysp]) >= 0 && $yyp < self::YYGLAST && self::$yygcheck[$yyp] == $yyn) { $yystate = self::$yygoto[$yyp]; } else { $yystate = self::$yygdefault[$yyn]; } ++$this->yysp; $yysstk[$this->yysp] = $yystate; $this->yyastk[$this->yysp] = $this->yyval; $yylstk[$this->yysp] = $yyline; $yydstk[$this->yysp] = $yyDC; } else { /* error */ throw new PHPParser_Error('Unexpected token ' . self::$yyterminals[$yychar], $yyline); } if ($yystate < self::YYNLSTATES) { break; } /* >= YYNLSTATES means shift-and-reduce */ $yyn = $yystate - self::YYNLSTATES; } } }
public function getNextToken(&$value = null, &$startAttributes = null, &$endAttributes = null) { $tokenId = parent::getNextToken($value, $startAttributes, $endAttributes); $startAttributes['startOffset'] = $endAttributes['endOffset'] = $this->pos; return $tokenId; }