/**
* Tokenizes the string
*
* @param string $string
* @param string $filename
*/
public function tokenize($string, $filename = null)
{
$this->data = $string;
$this->tokens = $this->lexer->tokenize($string, $filename);
$this->tokens[] = array('token' => self::TOKEN_EOS, 'value' => null, 'position' => null);
$this->cursor = -1;
$this->count = count($this->tokens);
}
/**
* Parse the program into a series of nodes.
*
* @param $in The program
* @return Node[]
*/
public function parse(Input $in)
{
$tokens = $this->lexer->tokenize($in);
$nodes = array();
while ($tokens->valid()) {
$nodes[] = $this->doParse($tokens);
}
return new Program($nodes);
}
/**
* Compile some CoffeeScript.
*
* Available options:
*
* 'filename' => The source file, for debugging (formatted into error messages)
* 'header' => Add a header to the generated source (default: TRUE)
* 'rewrite' => Enable rewriting token stream (debugging)
* 'tokens' => Reference to token stream (debugging)
* 'trace' => File to write parser trace to (debugging)
*
* @param string The source CoffeeScript code
* @param array Options (see above)
*
* @return string The resulting JavaScript (if there were no errors)
*/
static function compile($code, $options = array())
{
$lexer = new Lexer($code, $options);
if (isset($options['filename'])) {
Parser::$FILE = $options['filename'];
}
if (isset($options['tokens'])) {
$tokens =& $options['tokens'];
}
if (isset($options['trace'])) {
Parser::Trace(fopen($options['trace'], 'w', TRUE), '> ');
}
try {
$parser = new Parser();
foreach ($tokens = $lexer->tokenize() as $token) {
$parser->parse($token);
}
$js = $parser->parse(NULL)->compile($options);
} catch (\Exception $e) {
throw new Error("In {$options['filename']}, " . $e->getMessage());
}
if (!isset($options['header']) || $options['header']) {
$js = '// Generated by CoffeeScript PHP ' . VERSION . "\n" . $js;
}
return $js;
}
function buildForms($code)
{
$lexer = new Lexer();
$reader = new Reader();
$builder = new FormTreeBuilder();
$tokens = $lexer->tokenize($code);
$ast = $reader->parse($tokens);
return $builder->parseAst($ast);
}
public function translate($query)
{
$lexer = new Lexer();
$parser = new Parser();
$compiler = new Compiler($this->schema);
$tokens = $lexer->tokenize($query);
$request = $parser->parse($tokens);
return $compiler->compile($request);
}
private function dumpTokens($expression)
{
$lexer = new Lexer();
fwrite($this->out, "Tokens\n======\n\n");
$tokens = $lexer->tokenize($expression);
foreach ($tokens as $t) {
fprintf($this->out, "%3d %-13s %s\n", $t['pos'], $t['type'], json_encode($t['value']));
}
fwrite($this->out, "\n");
}
/**
* Process conditionals
*
* @param string $str The template string containing conditionals
* @param string $vars The variables to look for in the conditionals
* @return string The new template to use instead of $str.
*/
public function processConditionals($str, $vars)
{
$lexer = new Lexer();
// Get the token stream
$tokens = $lexer->tokenize($this->protectJavascript($str));
$parser = new Parser($tokens);
$parser->setVariables($this->prefixVariables($vars));
if ($this->safety === TRUE) {
$parser->safetyOn();
}
$output = $parser->parse();
return $this->unProtectJavascript($output);
}
/**
* Compile some CoffeeScript.
*
* @param $code The source CoffeeScript code.
* @param $options Compiler options.
*/
function compile($code, $options = array(), &$tokens = NULL)
{
$lexer = new Lexer($code, $options);
if (isset($options['file'])) {
Parser::$FILE = $options['file'];
}
if (isset($options['trace'])) {
Parser::Trace(fopen($options['trace'], 'w', TRUE), '> ');
}
$parser = new Parser();
foreach ($tokens = $lexer->tokenize() as $token) {
$parser->parse($token);
}
return $parser->parse(NULL)->compile($options);
}
/**
*
*/
public function testMustReplaceHtmlInlineTags()
{
$this->assertEquals("=<script></script>", $this->lexer->tokenize("<script></script>"));
}
function interpolate_string($str, array $options = array())
{
$options = array_merge(array('heredoc' => '', 'regex' => NULL), $options);
$tokens = array();
$pi = 0;
$i = -1;
while (isset($str[++$i])) {
$letter = $str[$i];
if ($letter === '\\') {
$i++;
continue;
}
if (!($letter === '#' && $str[$i + 1] === '{' && ($expr = $this->balanced_string(substr($str, $i + 1), '}')))) {
continue;
}
if ($pi < $i) {
$tokens[] = array('NEOSTRING', substr($str, $pi, $i - $pi));
}
$inner = substr($expr, 1, -1);
if (strlen($inner)) {
$lexer = new Lexer($inner, array('line' => $this->line, 'rewrite' => FALSE));
$nested = $lexer->tokenize();
array_pop($nested);
if (isset($nested[0]) && $nested[0][0] === t('TERMINATOR')) {
array_shift($nested);
}
if ($length = count($nested)) {
if ($length > 1) {
array_unshift($nested, array(t('('), '('));
$nested[] = array(t(')'), ')');
}
$tokens[] = array('TOKENS', $nested);
}
}
$i += strlen($expr);
$pi = $i + 1;
}
if ($i > $pi && $pi < strlen($str)) {
$tokens[] = array('NEOSTRING', substr($str, $pi));
}
if ($options['regex']) {
return $tokens;
}
if (!count($tokens)) {
return $this->token('STRING', '""');
}
if (!($tokens[0][0] === 'NEOSTRING')) {
array_unshift($tokens, array('', ''));
}
if ($interpolated = count($tokens) > 1) {
$this->token('(', '(');
}
for ($i = 0; $i < count($tokens); $i++) {
list($tag, $value) = $tokens[$i];
if ($i) {
$this->token('+', '+');
}
if ($tag === 'TOKENS') {
$this->tokens = array_merge($this->tokens, $value);
} else {
$this->token('STRING', $this->make_string($value, '"', $options['heredoc']));
}
}
if ($interpolated) {
$this->token(')', ')');
}
return $tokens;
}
/**
* @param string $string
*
* @return ExpressionResult
*/
public function compile($string)
{
return $this->parser->parse($this->lexer->tokenize($string));
}
public function isValid($template, &$error = null)
{
if (!is_string($template)) {
throw new \InvalidArgumentException('string expected');
}
$source = $this->options['source'];
$adapter = $this->options['adapter'];
$path = $this->resolvePath($template);
$class = self::CLASS_PREFIX . md5($path);
if (!$adapter->isReadable($path)) {
throw new \RuntimeException(sprintf('%s is not a valid readable template', $template));
}
try {
$lexer = new Lexer($adapter->getContents($path));
$parser = new Parser($lexer->tokenize());
$compiler = new Compiler($parser->parse());
} catch (\Exception $e) {
$error = $e->getMessage();
return false;
}
return true;
}
<?php
namespace Chubasco\ZeroCode;
// include shared code
include '../lib/common.php';
include '../lib/lexer.php';
include '../lib/token.php';
// start or continue the session
// session_start();
header('Cache-control: private');
$file = fopen("../README", "r");
while (!feof($file)) {
$linea = fgets($file);
// echo ">>".$linea. "<< ";
$current_token = new Token(Token::T_NONE, 0);
$tokens = Lexer::tokenize($linea);
// $tokens = Lexer::test($linea);
echo " <br />";
}
// wend
fclose($file);
/**
* @test
* @dataProvider provideTokenize
*/
public function tokenizeValue($expected, $code)
{
$lexer = new Lexer();
$this->assertSame($expected, $lexer->tokenize($code));
}
