function concat(TokenStream $ts) : TokenStream { $buffer = []; while ($t = $ts->current()) { $str = (string) $t; if (!preg_match('/^\\w+$/', $str)) { throw new YayException("Only valid identifiers are mergeable, '{$t->dump()}' given."); } $buffer[] = $str; $ts->next(); } return TokenStream::fromSequence(new Token(T_STRING, implode('', $buffer))); }
function testInject() { $ts = TokenStream::fromSource("<?php START END "); $ts->inject(TokenStream::fromEmpty()); $this->assertEquals('<?php START END ', (string) $ts); $ts->step(); $ts->step(); $ts->inject(TokenStream::fromSequence([T_STRING, 'MIDDLE', 0], [T_WHITESPACE, ' ', 0])); $this->assertEquals('<?php START MIDDLE END ', (string) $ts); $ts->reset(); $ts = TokenStream::fromEmpty(); $ts->inject(TokenStream::fromSequence([T_OPEN_TAG, '<?php', 0], [T_WHITESPACE, ' ', 0], [T_STRING, 'A', 0])); $this->assertEquals('<?php A', (string) $ts); }
private function mutate(TokenStream $ts, Ast $context, Cycle $cycle, Directives $directives, BlueContext $blueContext) : TokenStream { if ($this->constant) { return $ts; } static $states, $parser; $states = $states ?? new Stack(); $parser = $parser ?? traverse(token(Token::CLOAKED), consume(chain(rtoken('/^··\\w+$/')->as('expander'), either(parentheses(), braces())->as('args')))->onCommit(function (Ast $result) use($states) { $cg = $states->current(); $expander = $result->expander; if (\count($result->args) === 0) { $cg->this->fail(self::E_EMPTY_EXPANDER_SLICE, (string) $expander, $expander->line()); } $context = Map::fromKeysAndValues(['scope' => $cg->cycle->id(), 'directives' => $cg->directives, 'blueContext' => $cg->blueContext]); $expansion = TokenStream::fromSlice($result->args); $mutation = $cg->this->mutate(clone $expansion, $cg->context, $cg->cycle, $cg->directives, $cg->blueContext); $mutation = $cg->this->lookupExpander($expander)($mutation, $context); $cg->ts->inject($mutation); }), consume(chain(rtoken('/^·\\w+|···\\w+$/')->as('label'), operator('···'), optional(parentheses()->as('delimiters')), braces()->as('expansion')))->onCommit(function (Ast $result) use($states) { $cg = $states->current(); $context = $cg->this->lookupContext($result->label, $cg->context, self::E_UNDEFINED_EXPANSION); $expansion = TokenStream::fromSlice($result->expansion); $delimiters = $result->delimiters; // normalize single context if (array_values($context) !== $context) { $context = [$context]; } foreach (array_reverse($context) as $i => $subContext) { $mutation = $cg->this->mutate(clone $expansion, (new Ast(null, $subContext))->withParent($cg->context), $cg->cycle, $cg->directives, $cg->blueContext); if ($i !== 0) { foreach ($delimiters as $d) { $mutation->push($d); } } $cg->ts->inject($mutation); } }), consume(rtoken('/^(T_\\w+·\\w+|·\\w+|···\\w+)$/')->as('label'))->onCommit(function (Ast $result) use($states) { $cg = $states->current(); $context = $cg->this->lookupContext($result->label, $cg->context, self::E_UNDEFINED_EXPANSION); if ($context instanceof Token) { $cg->ts->inject(TokenStream::fromSequence($context)); } elseif (is_array($context) && \count($context)) { $tokens = []; array_walk_recursive($context, function (Token $token) use(&$tokens) { $tokens[] = $token; }); $cg->ts->inject(TokenStream::fromSlice($tokens)); } })); $cg = (object) ['ts' => $ts, 'context' => $context, 'directives' => $directives, 'cycle' => $cycle, 'this' => $this, 'blueContext' => $blueContext]; $states->push($cg); $parser->parse($cg->ts); $states->pop(); $cg->ts->reset(); if ($this->cloaked) { traverse(consume(token(Token::CLOAKED))->onCommit(function (Ast $result) use($cg) { $cg->ts->inject(TokenStream::fromSourceWithoutOpenTag((string) $result->token())); }))->parse($cg->ts); $cg->ts->reset(); } return $cg->ts; }
private function mutate(TokenStream $ts, Ast $context) : TokenStream { $cg = (object) ['ts' => clone $ts, 'context' => $context]; if ($this->unsafe && !$this->hasTag('·unsafe')) { hygienize($cg->ts, $this->cycle->id()); } if ($this->constant) { return $cg->ts; } traverse(either(token(Token::CLOAKED), consume(chain(rtoken('/^·\\w+$/')->as('expander'), parentheses()->as('args')))->onCommit(function (Ast $result) use($cg) { $expander = $this->lookupExpander($result->expander); $args = []; foreach ($result->args as $arg) { if ($arg instanceof Token) { $key = (string) $arg; if (preg_match('/^·\\w+|T_\\w+·\\w+|···\\w+$/', $key)) { $arg = $cg->context->{$key}; } } if (is_array($arg)) { array_push($args, ...$arg); } else { $args[] = $arg; } } $mutation = $expander(TokenStream::fromSlice($args), $this->cycle->id()); $cg->ts->inject($mutation); }), consume(chain(rtoken('/^·\\w+|···\\w+$/')->as('label'), operator('···'), braces()->as('expansion')))->onCommit(function (Ast $result) use($cg) { $index = (string) $result->label; $context = $cg->context->{$index}; if ($context === null) { $this->fail(self::E_EXPANSION, $index, $result->label->line(), json_encode(array_keys($cg->context->all()[0]), self::PRETTY_PRINT)); } $expansion = TokenStream::fromSlice($result->expansion); // normalize single context if (array_values($context) !== $context) { $context = [$context]; } foreach (array_reverse($context) as $i => $subContext) { $mutation = $this->mutate($expansion, (new Ast(null, $subContext))->withParent($cg->context)); $cg->ts->inject($mutation); } }), consume(rtoken('/^(T_\\w+·\\w+|·\\w+|···\\w+)$/'))->onCommit(function (Ast $result) use($cg) { $expansion = $cg->context->{(string) $result->token()}; if ($expansion instanceof Token) { $cg->ts->inject(TokenStream::fromSequence($expansion)); } elseif (is_array($expansion) && \count($expansion)) { $tokens = []; array_walk_recursive($expansion, function (Token $token) use(&$tokens) { $tokens[] = $token; }); $cg->ts->inject(TokenStream::fromSlice($tokens)); } }), any()))->parse($cg->ts); $cg->ts->reset(); if ($this->cloaked) { traverse(either(consume(token(Token::CLOAKED))->onCommit(function (Ast $result) use($cg) { $cg->ts->inject(TokenStream::fromSourceWithoutOpenTag((string) $result->token())); }), any()))->parse($cg->ts); $cg->ts->reset(); } return $cg->ts; }
function testInject() { $ts = TokenStream::fromSource('<?php START END'); $ts->next(); $ts->step(); $ts->inject(TokenStream::fromSequence([T_STRING, 'MIDDLE_B', 0], [T_WHITESPACE, ' ', 0])); $ts->inject(TokenStream::fromSequence([T_STRING, 'MIDDLE_A', 0], [T_WHITESPACE, ' ', 0])); $this->assertEquals('<?php START MIDDLE_A MIDDLE_B END', (string) $ts); $ts = TokenStream::fromSource(''); $ts->inject(TokenStream::fromSequence([T_WHITESPACE, ' ', 0], [T_STRING, 'BAR', 0], [T_WHITESPACE, ' ', 0])); $this->assertEquals(' BAR ', (string) $ts); $ts->inject(TokenStream::fromSequence([T_WHITESPACE, ' ', 0], [T_STRING, 'FOO', 0], [T_WHITESPACE, ' ', 0])); $this->assertEquals(' FOO BAR ', (string) $ts); $ts = TokenStream::fromSource('<?php A B'); $ts->next(); $ts->next(); $ts->next(); $node = $ts->index(); $partial = TokenStream::fromSequence([T_WHITESPACE, ' ', 0], [T_STRING, 'C', 0], [T_WHITESPACE, ' ', 0]); $index = $partial->index(); $ts->inject($partial); $this->assertEquals('<?php A B C ', (string) $ts); // $this->assertSame($index, $ts->index()); }