tokenize() public method

Tokenizes a source code.
public tokenize ( string $code, string $filename = 'n/a' ) : Twig_TokenStream
$code string The source code
$filename string A unique identifier for the source code
return Twig_TokenStream A token stream instance
Ejemplo n.º 1
0
 public function tokenize($code, $filename = null)
 {
     // Our phpBB tags
     // Commented out tokens are handled separately from the main replace
     $phpbb_tags = array('ENDDEFINE', 'INCLUDE', 'INCLUDEPHP', 'INCLUDEJS', 'INCLUDECSS', 'PHP', 'ENDPHP', 'EVENT');
     // Twig tag masks
     $twig_tags = array('autoescape', 'endautoescape', 'if', 'elseif', 'else', 'endif', 'block', 'endblock', 'use', 'extends', 'embed', 'filter', 'endfilter', 'flush', 'for', 'endfor', 'macro', 'endmacro', 'import', 'from', 'sandbox', 'endsandbox', 'set', 'endset', 'spaceless', 'endspaceless', 'verbatim', 'endverbatim');
     // Fix tokens that may have inline variables (e.g. <!-- DEFINE $TEST = '{FOO}')
     $code = $this->strip_surrounding_quotes(array('INCLUDE', 'INCLUDEPHP', 'INCLUDEJS', 'INCLUDECSS'), $code);
     $code = $this->fix_inline_variable_tokens(array('DEFINE \\$[a-zA-Z0-9_]+ =', 'INCLUDE', 'INCLUDEPHP', 'INCLUDEJS', 'INCLUDECSS'), $code);
     $code = $this->add_surrounding_quotes(array('INCLUDE', 'INCLUDEPHP', 'INCLUDEJS', 'INCLUDECSS'), $code);
     // Fix our BEGIN statements
     $code = $this->fix_begin_tokens($code);
     // Fix our IF tokens
     $code = $this->fix_if_tokens($code);
     // Fix our DEFINE tokens
     $code = $this->fix_define_tokens($code);
     // Replace all of our starting tokens, <!-- TOKEN --> with Twig style, {% TOKEN %}
     // This also strips outer parenthesis, <!-- IF (blah) --> becomes <!-- IF blah -->
     $code = preg_replace('#<!-- (' . implode('|', $phpbb_tags) . ')(?: (.*?) ?)?-->#', '{% $1 $2 %}', $code);
     // Replace all of our twig masks with Twig code (e.g. <!-- BLOCK .+ --> with {% block $1 %})
     $code = $this->replace_twig_tag_masks($code, $twig_tags);
     // Replace all of our language variables, {L_VARNAME}, with Twig style, {{ lang('NAME') }}
     // Appends any filters after lang()
     $code = preg_replace('#{L_([a-zA-Z0-9_\\.]+)(\\|[^}]+?)?}#', '{{ lang(\'$1\')$2 }}', $code);
     // Replace all of our escaped language variables, {LA_VARNAME}, with Twig style, {{ lang('NAME')|addslashes }}
     // Appends any filters after lang(), but before addslashes
     $code = preg_replace('#{LA_([a-zA-Z0-9_\\.]+)(\\|[^}]+?)?}#', '{{ lang(\'$1\')$2|addslashes }}', $code);
     // Replace all of our variables, {VARNAME}, with Twig style, {{ VARNAME }}
     // Appends any filters
     $code = preg_replace('#{([a-zA-Z0-9_\\.]+)(\\|[^}]+?)?}#', '{{ $1$2 }}', $code);
     return parent::tokenize($code, $filename);
 }
Ejemplo n.º 2
0
 protected function parseTwigFile($file)
 {
     $env = new \Twig_Environment(new \Twig_Loader_Filesystem(SOURCE_PATH), array('autoescape' => false, 'cache' => false, 'base_template_class' => 'eBuildy\\Templating\\TwigBaseTemplate'));
     $lexer = new \Twig_Lexer($env);
     $parser = new \Twig_Parser($env);
     $stream = $lexer->tokenize(file_get_contents($file), $file);
     while (!$stream->isEOF()) {
         $node = $stream->next();
         $method = $node->getValue();
         if ($method === 'getCss' || $method === 'addCss') {
             $type = 'css';
         } elseif ($method === 'getJs' || $method === 'addJs') {
             $type = 'js';
         } else {
             $type = null;
         }
         if ($type !== null) {
             $arg = $this->getTwigArgument($stream);
             $sourcePath = AssetResolver::resolveSourcePath($arg, \eBuildy\Helper\ResolverHelper::getModulePathFromView($file));
             if ($sourcePath === null) {
                 // var_dump(array('file' => $file, 'type' => $type, 'source' => $arg));
             } else {
                 $this->assets[$sourcePath] = $type;
             }
         }
     }
 }
Ejemplo n.º 3
0
 public function testLongBlock1()
 {
     $template = '{% ' . str_repeat('*', 100000) . ' %}';
     $lexer = new Twig_Lexer(new Twig_Environment());
     $stream = $lexer->tokenize($template);
     // should not throw an exception
 }
 public function testLineDirectiveInline()
 {
     $template = "foo\n" . "bar{% line 10 %}{{\n" . "baz\n" . "}}\n";
     $lexer = new Twig_Lexer(new Twig_Environment());
     $stream = $lexer->tokenize($template);
     // foo\nbar
     $this->assertSame(1, $stream->expect(Twig_Token::TEXT_TYPE)->getLine());
     // {{
     $this->assertSame(10, $stream->expect(Twig_Token::VAR_START_TYPE)->getLine());
     // baz
     $this->assertSame(11, $stream->expect(Twig_Token::NAME_TYPE)->getLine());
 }
Ejemplo n.º 5
0
 protected function countToken($template, $type, $value = null)
 {
     $lexer = new Twig_Lexer(new Twig_Environment());
     $stream = $lexer->tokenize($template);
     $count = 0;
     $tokens = array();
     while (!$stream->isEOF()) {
         $token = $stream->next();
         if ($type === $token->getType()) {
             if (null === $value || $value === $token->getValue()) {
                 ++$count;
             }
         }
     }
     return $count;
 }
Ejemplo n.º 6
0
    /**
     * @expectedException Twig_Error_Syntax
     * @expectedExceptionMessage Unclosed "block" at line 3
     */
    public function testUnterminatedBlock()
    {
        $template = '

{%

bar


';
        $lexer = new Twig_Lexer(new Twig_Environment());
        $lexer->tokenize($template);
    }
Ejemplo n.º 7
0
    /**
     * @expectedException Twig_Error_Syntax
     * @expectedExceptionMessage Unclosed "block" at line 3
     */
    public function testUnterminatedBlock()
    {
        $template = '

{%

bar


';
        $lexer = new Twig_Lexer(new Twig_Environment($this->getMock('Twig_LoaderInterface')));
        $lexer->tokenize($template);
    }
Ejemplo n.º 8
0
 /**
  * Overrides tokenize to initialize $this->commentTokens.
  */
 public function tokenize($code, $filename = null)
 {
     $this->commentTokens = array();
     return parent::tokenize($code, $filename);
 }
Ejemplo n.º 9
0
 /**
  * @return \Twig_TokenStream
  * @throws \Twig_Error_Syntax
  */
 protected function getTemplateTokens()
 {
     $template = file_get_contents($this->file);
     $lexer = new \Twig_Lexer(new \Twig_Environment());
     return $lexer->tokenize($template);
 }
Ejemplo n.º 10
0
 public function testOperatorEndingWithALetterAtTheEndOfALine()
 {
     $template = "{{ 1 and\n0}}";
     $lexer = new Twig_Lexer(new Twig_Environment());
     $stream = $lexer->tokenize($template);
     $stream->expect(Twig_Token::VAR_START_TYPE);
     $stream->expect(Twig_Token::NUMBER_TYPE, 1);
     $stream->expect(Twig_Token::OPERATOR_TYPE, 'and');
 }
Ejemplo n.º 11
0
<?php

require __DIR__ . '/../vendor/autoload.php';
$env = new Twig_Environment();
$lexer = new Twig_Lexer($env);
$tokens = $lexer->tokenize("Hello {{ aaa }} {% block right %} {% endblock %}");
$module = $env->parse($tokens);
$class = new ReflectionClass(Twig_Node::class);
$attr_field = $class->getProperty("attributes");
$attr_field->setAccessible(true);
$nodes_field = $class->getProperty("nodes");
$nodes_field->setAccessible(true);
echo $module->__toString();