public function tokenize($code, $filename = null)
{
// Our phpBB tags
// Commented out tokens are handled separately from the main replace
$phpbb_tags = array('ENDDEFINE', 'INCLUDE', 'INCLUDEPHP', 'INCLUDEJS', 'INCLUDECSS', 'PHP', 'ENDPHP', 'EVENT');
// Twig tag masks
$twig_tags = array('autoescape', 'endautoescape', 'if', 'elseif', 'else', 'endif', 'block', 'endblock', 'use', 'extends', 'embed', 'filter', 'endfilter', 'flush', 'for', 'endfor', 'macro', 'endmacro', 'import', 'from', 'sandbox', 'endsandbox', 'set', 'endset', 'spaceless', 'endspaceless', 'verbatim', 'endverbatim');
// Fix tokens that may have inline variables (e.g. <!-- DEFINE $TEST = '{FOO}')
$code = $this->strip_surrounding_quotes(array('INCLUDE', 'INCLUDEPHP', 'INCLUDEJS', 'INCLUDECSS'), $code);
$code = $this->fix_inline_variable_tokens(array('DEFINE \\$[a-zA-Z0-9_]+ =', 'INCLUDE', 'INCLUDEPHP', 'INCLUDEJS', 'INCLUDECSS'), $code);
$code = $this->add_surrounding_quotes(array('INCLUDE', 'INCLUDEPHP', 'INCLUDEJS', 'INCLUDECSS'), $code);
// Fix our BEGIN statements
$code = $this->fix_begin_tokens($code);
// Fix our IF tokens
$code = $this->fix_if_tokens($code);
// Fix our DEFINE tokens
$code = $this->fix_define_tokens($code);
// Replace all of our starting tokens, <!-- TOKEN --> with Twig style, {% TOKEN %}
// This also strips outer parenthesis, <!-- IF (blah) --> becomes <!-- IF blah -->
$code = preg_replace('#<!-- (' . implode('|', $phpbb_tags) . ')(?: (.*?) ?)?-->#', '{% $1 $2 %}', $code);
// Replace all of our twig masks with Twig code (e.g. <!-- BLOCK .+ --> with {% block $1 %})
$code = $this->replace_twig_tag_masks($code, $twig_tags);
// Replace all of our language variables, {L_VARNAME}, with Twig style, {{ lang('NAME') }}
// Appends any filters after lang()
$code = preg_replace('#{L_([a-zA-Z0-9_\\.]+)(\\|[^}]+?)?}#', '{{ lang(\'$1\')$2 }}', $code);
// Replace all of our escaped language variables, {LA_VARNAME}, with Twig style, {{ lang('NAME')|addslashes }}
// Appends any filters after lang(), but before addslashes
$code = preg_replace('#{LA_([a-zA-Z0-9_\\.]+)(\\|[^}]+?)?}#', '{{ lang(\'$1\')$2|addslashes }}', $code);
// Replace all of our variables, {VARNAME}, with Twig style, {{ VARNAME }}
// Appends any filters
$code = preg_replace('#{([a-zA-Z0-9_\\.]+)(\\|[^}]+?)?}#', '{{ $1$2 }}', $code);
return parent::tokenize($code, $filename);
}
protected function parseTwigFile($file)
{
$env = new \Twig_Environment(new \Twig_Loader_Filesystem(SOURCE_PATH), array('autoescape' => false, 'cache' => false, 'base_template_class' => 'eBuildy\\Templating\\TwigBaseTemplate'));
$lexer = new \Twig_Lexer($env);
$parser = new \Twig_Parser($env);
$stream = $lexer->tokenize(file_get_contents($file), $file);
while (!$stream->isEOF()) {
$node = $stream->next();
$method = $node->getValue();
if ($method === 'getCss' || $method === 'addCss') {
$type = 'css';
} elseif ($method === 'getJs' || $method === 'addJs') {
$type = 'js';
} else {
$type = null;
}
if ($type !== null) {
$arg = $this->getTwigArgument($stream);
$sourcePath = AssetResolver::resolveSourcePath($arg, \eBuildy\Helper\ResolverHelper::getModulePathFromView($file));
if ($sourcePath === null) {
// var_dump(array('file' => $file, 'type' => $type, 'source' => $arg));
} else {
$this->assets[$sourcePath] = $type;
}
}
}
}
public function testLongBlock1()
{
$template = '{% ' . str_repeat('*', 100000) . ' %}';
$lexer = new Twig_Lexer(new Twig_Environment());
$stream = $lexer->tokenize($template);
// should not throw an exception
}
public function testLineDirectiveInline()
{
$template = "foo\n" . "bar{% line 10 %}{{\n" . "baz\n" . "}}\n";
$lexer = new Twig_Lexer(new Twig_Environment());
$stream = $lexer->tokenize($template);
// foo\nbar
$this->assertSame(1, $stream->expect(Twig_Token::TEXT_TYPE)->getLine());
// {{
$this->assertSame(10, $stream->expect(Twig_Token::VAR_START_TYPE)->getLine());
// baz
$this->assertSame(11, $stream->expect(Twig_Token::NAME_TYPE)->getLine());
}
protected function countToken($template, $type, $value = null)
{
$lexer = new Twig_Lexer(new Twig_Environment());
$stream = $lexer->tokenize($template);
$count = 0;
$tokens = array();
while (!$stream->isEOF()) {
$token = $stream->next();
if ($type === $token->getType()) {
if (null === $value || $value === $token->getValue()) {
++$count;
}
}
}
return $count;
}
/**
* @expectedException Twig_Error_Syntax
* @expectedExceptionMessage Unclosed "block" at line 3
*/
public function testUnterminatedBlock()
{
$template = '
{%
bar
';
$lexer = new Twig_Lexer(new Twig_Environment());
$lexer->tokenize($template);
}
/**
* @expectedException Twig_Error_Syntax
* @expectedExceptionMessage Unclosed "block" at line 3
*/
public function testUnterminatedBlock()
{
$template = '
{%
bar
';
$lexer = new Twig_Lexer(new Twig_Environment($this->getMock('Twig_LoaderInterface')));
$lexer->tokenize($template);
}
/**
* Overrides tokenize to initialize $this->commentTokens.
*/
public function tokenize($code, $filename = null)
{
$this->commentTokens = array();
return parent::tokenize($code, $filename);
}
/**
* @return \Twig_TokenStream
* @throws \Twig_Error_Syntax
*/
protected function getTemplateTokens()
{
$template = file_get_contents($this->file);
$lexer = new \Twig_Lexer(new \Twig_Environment());
return $lexer->tokenize($template);
}
public function testOperatorEndingWithALetterAtTheEndOfALine()
{
$template = "{{ 1 and\n0}}";
$lexer = new Twig_Lexer(new Twig_Environment());
$stream = $lexer->tokenize($template);
$stream->expect(Twig_Token::VAR_START_TYPE);
$stream->expect(Twig_Token::NUMBER_TYPE, 1);
$stream->expect(Twig_Token::OPERATOR_TYPE, 'and');
}
<?php
require __DIR__ . '/../vendor/autoload.php';
$env = new Twig_Environment();
$lexer = new Twig_Lexer($env);
$tokens = $lexer->tokenize("Hello {{ aaa }} {% block right %} {% endblock %}");
$module = $env->parse($tokens);
$class = new ReflectionClass(Twig_Node::class);
$attr_field = $class->getProperty("attributes");
$attr_field->setAccessible(true);
$nodes_field = $class->getProperty("nodes");
$nodes_field->setAccessible(true);
echo $module->__toString();