public static function format($code)
{
// Haskell lexer goes here
global $haskell_lang;
$lex = new Lexer($haskell_lang, $code);
$out = '';
while (!$lex->end()) {
list($type, $match) = $lex->next();
$is_html = false;
if ($type == '!!!notation') {
list($type, $next_match) = $lex->next();
$match = substr($match, 5, -5);
$is_html = true;
}
if ($type == '') {
$out .= htmlspecialchars($match);
} elseif ($type == '!!!') {
$out .= substr($match, 3, -3);
} else {
if (!$is_html) {
$match = htmlspecialchars($match);
}
if ($type == 'keyword') {
$match = preg_replace('@^__keyword__@', '', $match);
}
if ($type == 'varid' || $type == 'varop' || $type == 'comment') {
$match = preg_replace('@__([[:alnum:]_]+)@', '<sub>\\1</sub>', $match);
$match = preg_replace('@__[{]([^}]*)[}]@', '<sub>\\1</sub>', $match);
$match = preg_replace('@!!!(.*?)!!!@e', 'htmlspecialchars_decode("\\1")', $match);
}
$out .= "<span class=\"{$type}\">{$match}</span>";
}
}
return $out;
}
/**
* Compile some CoffeeScript.
*
* Available options:
*
* 'filename' => The source file, for debugging (formatted into error messages)
* 'header' => Add a header to the generated source (default: TRUE)
* 'rewrite' => Enable rewriting token stream (debugging)
* 'tokens' => Reference to token stream (debugging)
* 'trace' => File to write parser trace to (debugging)
*
* @param string The source CoffeeScript code
* @param array Options (see above)
*
* @return string The resulting JavaScript (if there were no errors)
*/
static function compile($code, $options = array())
{
$lexer = new Lexer($code, $options);
if (isset($options['filename'])) {
Parser::$FILE = $options['filename'];
}
if (isset($options['tokens'])) {
$tokens =& $options['tokens'];
}
if (isset($options['trace'])) {
Parser::Trace(fopen($options['trace'], 'w', TRUE), '> ');
}
try {
$parser = new Parser();
foreach ($tokens = $lexer->tokenize() as $token) {
$parser->parse($token);
}
$js = $parser->parse(NULL)->compile($options);
} catch (\Exception $e) {
throw new Error("In {$options['filename']}, " . $e->getMessage());
}
if (!isset($options['header']) || $options['header']) {
$js = '// Generated by CoffeeScript PHP ' . VERSION . "\n" . $js;
}
return $js;
}
/**
* @param Lexer $lexer
* @param char $char
*/
public function input(Lexer $lexer, $char)
{
if ($char == '/') {
$lexer->pushPath(new PathElementParent());
} else {
$this->throwError($lexer, $char);
}
}
/**
* Tokenizes the string
*
* @param string $string
* @param string $filename
*/
public function tokenize($string, $filename = null)
{
$this->data = $string;
$this->tokens = $this->lexer->tokenize($string, $filename);
$this->tokens[] = array('token' => self::TOKEN_EOS, 'value' => null, 'position' => null);
$this->cursor = -1;
$this->count = count($this->tokens);
}
/**
* @dataProvider provideTestHaltCompiler
*/
public function testHandleHaltCompiler($code, $remaining)
{
$this->lexer->startLexing($code);
while (Parser::T_HALT_COMPILER !== $this->lexer->getNextToken()) {
}
$this->assertEquals($this->lexer->handleHaltCompiler(), $remaining);
$this->assertEquals(0, $this->lexer->getNextToken());
}
/**
* @param string $tplName
* @param string $source
* @return string target codes
*/
public function compile($tplName, $source)
{
$lexer = new Lexer();
$tokenStream = $lexer->lex($source, $tplName);
$parser = new Parser\Parser();
$nodeTree = $parser->parse($tokenStream);
return $nodeTree->compile();
}
/**
* Parse the program into a series of nodes.
*
* @param $in The program
* @return Node[]
*/
public function parse(Input $in)
{
$tokens = $this->lexer->tokenize($in);
$nodes = array();
while ($tokens->valid()) {
$nodes[] = $this->doParse($tokens);
}
return new Program($nodes);
}
function buildForms($code)
{
$lexer = new Lexer();
$reader = new Reader();
$builder = new FormTreeBuilder();
$tokens = $lexer->tokenize($code);
$ast = $reader->parse($tokens);
return $builder->parseAst($ast);
}
public function translate($query)
{
$lexer = new Lexer();
$parser = new Parser();
$compiler = new Compiler($this->schema);
$tokens = $lexer->tokenize($query);
$request = $parser->parse($tokens);
return $compiler->compile($request);
}
private function dumpTokens($expression)
{
$lexer = new Lexer();
fwrite($this->out, "Tokens\n======\n\n");
$tokens = $lexer->tokenize($expression);
foreach ($tokens as $t) {
fprintf($this->out, "%3d %-13s %s\n", $t['pos'], $t['type'], json_encode($t['value']));
}
fwrite($this->out, "\n");
}
public function run()
{
$Attendees = array();
$lexer = new Lexer(new LexerConfig());
$interpreter = new Interpreter();
$interpreter->addObserver(function (array $row) use($Attendees) {
$ItemOne = \Todo\Attendee::create(array('company' => $row[0], 'lastname' => $row[1], 'firstname' => $row[2], 'phone' => $row[3], 'email' => $row[4], 'paidinfull' => $row[5], 'balance' => $row[6], 'notes' => $row[7], 'seat_id' => $row[8]));
$this->command->info("Added " . $row[1] . "as an attendee");
});
$fileName = $this->getDir() . '/Attendees.csv';
$lexer->parse($fileName, $interpreter);
}
/**
* @param Lexer $lexer
* @param char $char
*/
public function input(Lexer $lexer, $char)
{
if ($char == '=') {
$this->buffer .= $char;
} else {
if (self::isBlank($char)) {
$lexer->pushOperator(new TokenComparisonBinop($this->buffer));
} else {
$lexer->pushOperator(new TokenComparisonBinop($this->buffer));
$lexer->forwardInput($char);
}
}
}
public function run()
{
$this->input = filter_input(INPUT_POST, 'english', FILTER_SANITIZE_FULL_SPECIAL_CHARS);
if (isset($this->input)) {
$lexer = new Lexer($this->input);
$tokens = $lexer->run();
$translator = new Translator($tokens);
$result = $translator->run();
if ($this->isAjax()) {
$this->sendPayload($result);
}
}
}
/**
* Process conditionals
*
* @param string $str The template string containing conditionals
* @param string $vars The variables to look for in the conditionals
* @return string The new template to use instead of $str.
*/
public function processConditionals($str, $vars)
{
$lexer = new Lexer();
// Get the token stream
$tokens = $lexer->tokenize($this->protectJavascript($str));
$parser = new Parser($tokens);
$parser->setVariables($this->prefixVariables($vars));
if ($this->safety === TRUE) {
$parser->safetyOn();
}
$output = $parser->parse();
return $this->unProtectJavascript($output);
}
public function testScannerTokenizesSimpleXPathCorrectly()
{
$xpath = '//bookstore/book';
$lexer = new Lexer($xpath);
$tokens = [['value' => '//', 'type' => Lexer::T_NODE_SEARCH_SEPARATOR, 'position' => 0], ['value' => 'bookstore', 'type' => Lexer::T_NODE_NAME, 'position' => 2], ['value' => '/', 'type' => Lexer::T_NODESEPARATOR, 'position' => 12], ['value' => 'book', 'type' => Lexer::T_NODE_NAME, 'position' => 13]];
foreach ($tokens as $expected) {
$lexer->moveNext();
$actual = $lexer->lookahead;
$this->assertEquals($expected['value'], $actual['value']);
$this->assertEquals($expected['type'], $actual['type']);
$this->assertEquals($expected['position'], $actual['position']);
}
$this->assertFalse($lexer->moveNext());
}
/**
* @param Lexer $lexer
* @param char $char
*/
public function input(Lexer $lexer, $char)
{
if ($this->escaping) {
$this->buffer .= $char;
$this->escaping = false;
} else {
if ($char == "'") {
$lexer->pushOperand(new TokenLiteral($this->buffer));
$this->closed = true;
} else {
if ($char == '\\') {
$this->escaping = true;
} else {
if ($this->closed) {
if ($char == ',') {
$lexer->incrementLastFunctionArity();
$lexer->setStateEmpty();
} else {
if ($char == ')') {
$lexer->closeParenthesis();
} else {
if (self::isBlank($char)) {
} else {
if ($char == '+' || $char == '-') {
$lexer->pushOperator(new TokenPlusMinus($char));
} else {
if ($char == '=' || $char == '!') {
$lexer->setStateComparison($char);
} else {
if ($char == '*') {
$lexer->pushOperator(new TokenMul());
} else {
if ($char == ']') {
$lexer->closeSquareBracket();
} else {
$this->throwError($lexer, $char);
}
}
}
}
}
}
}
} else {
$this->buffer .= $char;
}
}
}
}
}
/**
* Compile some CoffeeScript.
*
* @param $code The source CoffeeScript code.
* @param $options Compiler options.
*/
function compile($code, $options = array(), &$tokens = NULL)
{
$lexer = new Lexer($code, $options);
if (isset($options['file'])) {
Parser::$FILE = $options['file'];
}
if (isset($options['trace'])) {
Parser::Trace(fopen($options['trace'], 'w', TRUE), '> ');
}
$parser = new Parser();
foreach ($tokens = $lexer->tokenize() as $token) {
$parser->parse($token);
}
return $parser->parse(NULL)->compile($options);
}
public function postUpload($unique)
{
//disable php timeout and db query log
DB::disableQueryLog();
set_time_limit(0);
$compare = Compare::where('unique', $unique)->first();
if (!$compare || !Session::get('owner')) {
return Redirect::to('compare/notfound');
}
$file = Input::file('csv');
$data_set = array();
$config = new LexerConfig();
$config->setDelimiter("\t");
$lexer = new Lexer($config);
//setup the interpreter
$interpreter = new Interpreter();
$interpreter->unstrict();
$interpreter->addObserver(function (array $row) use(&$data_set) {
$data_set[] = $row[0];
});
//parse it!
$lexer->parse($file, $interpreter);
//loop through the data
foreach ($data_set as $data_item) {
$data = new Data();
$data->compare_id = $unique;
$data->owner_id = Session::get('owner');
$data->hash = md5($data_item);
$data->save();
}
return Redirect::back();
}
private function getLink()
{
$link = '';
if (substr($this->lexer->lookahead['value'], 1, 1) == '[' && substr($this->lexer->lookahead['value'], -1, 1) == ']') {
//We are looking for a link like $h[xxx]yyy$h
$this->lexer->moveNext();
$matches = array();
if ($this->lexer->lookahead && preg_match('/^\\[([^\\]]*)\\]$/iu', $this->lexer->lookahead['value'], $matches)) {
$link = $matches[1];
}
} else {
//We are looking for a link like $hxxxx$h
$endLinkTokens = array(Lexer::T_EXTERNAL_LINK, Lexer::T_INTERNAL_LINK, Lexer::T_RESET_ALL);
do {
$nextLookahead = $this->lexer->peek();
if ($nextLookahead && ($nextLookahead['type'] == Lexer::T_NONE || $nextLookahead['type'] == Lexer::T_ESCAPED_CHAR)) {
$link .= $nextLookahead['value'];
if (substr($link, 0, 1) == '[') {
//It means that there is no closing square bracket $l[noclosingsquarebracket
array_push($this->lookaheadToSkip, $nextLookahead);
}
}
} while ($nextLookahead !== null && !in_array($nextLookahead['type'], $endLinkTokens));
if (substr($link, 0, 1) == '[') {
//It means that there is no closing square bracket $l[noclosingsquarebracket
$link = '';
}
}
return $link;
}
/**
* Throw a syntax error
*
* @param string $expected
* @throws ParserException
*/
private function syntaxError($expected)
{
$token = $this->lexer->lookahead;
$tokenPosition = isset($token['position']) ? $token['position'] : '-1';
$message = sprintf('line 0, col %s: Error: Expected %s, got ', $tokenPosition, $expected);
if ($this->lexer->lookahead === null) {
$message .= 'end of string';
} else {
$message .= $this->lexer->getLiteral($token['type']);
}
throw new ParserException($message);
}
/**
* @param Lexer $lexer
* @param char $char
*/
public function input(Lexer $lexer, $char)
{
if (self::isBlank($char)) {
} else {
if ($char == '+' || $char == '-') {
$lexer->pushOperator(new TokenPlusMinus($char));
} else {
if ($char == '*') {
$lexer->pushOperator(new TokenMul());
} else {
if ($char == '!') {
$lexer->setStateComparison($char);
} else {
if ($char == '=') {
$lexer->setStateComparison($char);
} else {
if (self::isAlpha($char)) {
$lexer->setStateSymbol($char);
} else {
if ($char == ')') {
$lexer->closeParenthesis();
} else {
if ($char == ']') {
$lexer->closeSquareBracket();
} else {
if ($char == ',') {
$lexer->incrementLastFunctionArity();
} else {
$this->throwError($lexer, $char);
}
}
}
}
}
}
}
}
}
}
/**
* Creates a new B8 object.
*
* @param array custom configuration
* @return void
*/
public function __construct(array $config = array())
{
// Load the lexer default config file
$this->config = Kohana::$config->load('b8');
// Overwrite with custom config settings
foreach ($config as $key => $value) {
$this->config[$key] = $value;
}
// Setup lexer, passing config settings through in case of custom configuration
$this->lexer = Lexer::factory($this->config['use_lexer'], $this->config['lexer'][$this->config['use_lexer']]);
// Setup storage, passing config settings through in case of custom configuration
$this->storage = Storage::factory($this->config['use_storage'], $this->config['storage'][$this->config['use_storage']]);
}
/**
* @param Lexer $lexer
* @param char $char
*/
public function input(Lexer $lexer, $char)
{
if ($char == ')') {
$lexer->pushOperator(new TokenFunction($this->buffer, 0));
$lexer->closeParenthesis();
} else {
if ($char == '(') {
$lexer->pushOperator(new TokenFunction($this->buffer, 1));
$lexer->pushOperator(new TokenLParen());
} else {
if (self::isBlank($char)) {
} else {
if ($char == "'") {
$lexer->pushOperator(new TokenFunction($this->buffer, 1));
$lexer->setStateString();
} else {
$lexer->pushOperator(new TokenFunction($this->buffer, 1));
$lexer->forwardInput($char);
}
}
}
}
}
private static function initTokenMap()
{
if (!self::$tokenMap) {
self::$tokenMap = array();
// 256 is the minimum possible token number, as everything below
// it is an ASCII value
for ($i = 256; $i < 1000; ++$i) {
// T_DOUBLE_COLON is equivalent to T_PAAMAYIM_NEKUDOTAYIM
if (T_DOUBLE_COLON === $i) {
self::$tokenMap[$i] = Parser::T_PAAMAYIM_NEKUDOTAYIM;
// T_OPEN_TAG_WITH_ECHO with dropped T_OPEN_TAG results in T_ECHO
} elseif (T_OPEN_TAG_WITH_ECHO === $i) {
self::$tokenMap[$i] = Parser::T_ECHO;
// T_CLOSE_TAG is equivalent to ';'
} elseif (T_CLOSE_TAG === $i) {
self::$tokenMap[$i] = ord(';');
// and the others can be mapped directly
} elseif ('UNKNOWN' !== ($name = token_name($i))) {
self::$tokenMap[$i] = constant('Parser::' . $name);
}
}
}
}
public function __construct($input)
{
parent::__construct($input);
}
/**
* Decode hip data string to an array
*
* @throws Hip\Exception
*
* @param string $hip
* @return array
*/
public static function decode($hip)
{
$lexer = new Lexer($hip);
$parser = new Parser($lexer->tokens());
return $parser->parse();
}
/**
*
*/
public function testMustReplaceHtmlInlineTags()
{
$this->assertEquals("=<script></script>", $this->lexer->tokenize("<script></script>"));
}
function interpolate_string($str, array $options = array())
{
$options = array_merge(array('heredoc' => '', 'regex' => NULL), $options);
$tokens = array();
$pi = 0;
$i = -1;
while (isset($str[++$i])) {
$letter = $str[$i];
if ($letter === '\\') {
$i++;
continue;
}
if (!($letter === '#' && $str[$i + 1] === '{' && ($expr = $this->balanced_string(substr($str, $i + 1), '}')))) {
continue;
}
if ($pi < $i) {
$tokens[] = array('NEOSTRING', substr($str, $pi, $i - $pi));
}
$inner = substr($expr, 1, -1);
if (strlen($inner)) {
$lexer = new Lexer($inner, array('line' => $this->line, 'rewrite' => FALSE));
$nested = $lexer->tokenize();
array_pop($nested);
if (isset($nested[0]) && $nested[0][0] === t('TERMINATOR')) {
array_shift($nested);
}
if ($length = count($nested)) {
if ($length > 1) {
array_unshift($nested, array(t('('), '('));
$nested[] = array(t(')'), ')');
}
$tokens[] = array('TOKENS', $nested);
}
}
$i += strlen($expr);
$pi = $i + 1;
}
if ($i > $pi && $pi < strlen($str)) {
$tokens[] = array('NEOSTRING', substr($str, $pi));
}
if ($options['regex']) {
return $tokens;
}
if (!count($tokens)) {
return $this->token('STRING', '""');
}
if (!($tokens[0][0] === 'NEOSTRING')) {
array_unshift($tokens, array('', ''));
}
if ($interpolated = count($tokens) > 1) {
$this->token('(', '(');
}
for ($i = 0; $i < count($tokens); $i++) {
list($tag, $value) = $tokens[$i];
if ($i) {
$this->token('+', '+');
}
if ($tag === 'TOKENS') {
$this->tokens = array_merge($this->tokens, $value);
} else {
$this->token('STRING', $this->make_string($value, '"', $options['heredoc']));
}
}
if ($interpolated) {
$this->token(')', ')');
}
return $tokens;
}
/**
* @param string $body
* @return Token
*/
private function lexOne($body)
{
$lexer = new Lexer(new Source($body));
return $lexer->nextToken();
}
/**
* Generates the lexer cache on fly.
*
* @param \FSHL\Lexer $lexer
*
* @return \FSHL\Highlighter
*/
private function generateCache(Lexer $lexer)
{
$generator = new Generator($lexer);
try {
$generator->saveToCache();
} catch (\RuntimeException $e) {
$file = tempnam(sys_get_temp_dir(), 'fshl');
file_put_contents($file, $generator->getSource());
require_once $file;
unlink($file);
}
$lexerName = $lexer->getLanguage();
$lexerCacheClass = '\\FSHL\\Lexer\\Cache\\' . $lexerName;
$this->lexers[$lexerName] = new $lexerCacheClass();
$this->lexer = $this->lexers[$lexerName];
return $this;
}
