private function compile(array $tokens)
{
$cg = (object) ['ts' => TokenStream::fromSlice($tokens), 'parsers' => []];
traverse(rtoken('/^(T_\\w+)·(\\w+)$/')->onCommit(function (Ast $result) use($cg) {
$token = $result->token();
$id = $this->lookupCapture($token);
$type = $this->lookupTokenType($token);
$cg->parsers[] = token($type)->as($id);
}), ($parser = chain(rtoken('/^·\\w+$/')->as('type'), token('('), optional(ls(either(future($parser)->as('parser'), chain(token(T_FUNCTION), parentheses()->as('args'), braces()->as('body'))->as('function'), string()->as('string'), rtoken('/^T_\\w+·\\w+$/')->as('token'), rtoken('/^T_\\w+$/')->as('constant'), rtoken('/^·this$/')->as('this'), label()->as('label'))->as('parser'), token(',')))->as('args'), commit(token(')')), optional(rtoken('/^·\\w+$/')->as('label'), null)))->onCommit(function (Ast $result) use($cg) {
$cg->parsers[] = $this->compileParser($result->type, $result->args, $result->label);
}), $this->layer('{', '}', braces(), $cg), $this->layer('[', ']', brackets(), $cg), $this->layer('(', ')', parentheses(), $cg), rtoken('/^···(\\w+)$/')->onCommit(function (Ast $result) use($cg) {
$id = $this->lookupCapture($result->token());
$cg->parsers[] = layer()->as($id);
}), token(T_STRING, '·')->onCommit(function (Ast $result) use($cg) {
$offset = \count($cg->parsers);
if (0 !== $this->dominance || 0 === $offset) {
$this->fail(self::E_BAD_DOMINANCE, $offset, $result->token()->line());
}
$this->dominance = $offset;
}), rtoken('/·/')->onCommit(function (Ast $result) {
$token = $result->token();
$this->fail(self::E_BAD_CAPTURE, $token, $token->line());
}), any()->onCommit(function (Ast $result) use($cg) {
$cg->parsers[] = token($result->token());
}))->parse($cg->ts);
// check if macro dominance '·' is last token
if ($this->dominance === \count($cg->parsers)) {
$this->fail(self::E_BAD_DOMINANCE, $this->dominance, $cg->ts->last()->line());
}
$this->specificity = \count($cg->parsers);
if ($this->specificity > 1) {
if (0 === $this->dominance) {
$pattern = chain(...$cg->parsers);
} else {
/*
dominat macros are partially wrapped in commit()s and dominance
is the offset used as the 'event horizon' point... once the entry
point is matched, there is no way back and a parser error arises
*/
$prefix = array_slice($cg->parsers, 0, $this->dominance);
$suffix = array_slice($cg->parsers, $this->dominance);
$pattern = chain(...array_merge($prefix, array_map(commit::class, $suffix)));
}
} else {
/*
micro optimization to save one function call for every token on the subject
token stream whenever the macro pattern consists of a single parser
*/
$pattern = $cg->parsers[0];
}
return $pattern;
}
function hygienize(TokenStream $ts, string $scope) : TokenStream
{
$ts->reset();
traverse(either(chain(token(T_STRING, '·unsafe'), parentheses()), either(token(T_VARIABLE)->as('target'), chain(identifier()->as('target'), token(':')), chain(token(T_GOTO), identifier()->as('target')))->onCommit(function (Ast $result) use($scope) {
(function () use($scope) {
if ((string) $this !== '$this') {
$this->value = (string) $this . '·' . $scope;
}
})->call($result->target);
}), any()))->parse($ts);
$ts->reset();
return $ts;
}
function hygienize(TokenStream $ts, array $context) : TokenStream
{
$ts->reset();
$cg = (object) ['node' => null, 'context' => $context, 'ts' => $ts];
$saveNode = function (Parser $parser) use($cg) {
return midrule(function ($ts) use($cg, $parser) {
$cg->node = $ts->index();
return $parser->parse($ts);
});
};
traverse(chain(token(T_STRING, '··unsafe'), either(parentheses(), braces())), either($saveNode(token(T_VARIABLE)), chain($saveNode(identifier()), token(':')), chain(token(T_GOTO), $saveNode(identifier())))->onCommit(function (Ast $result) use($cg) {
if (($t = $cg->node->token) && ($value = (string) $t) !== '$this') {
$cg->node->token = new Token($t->type(), "{$value}·{$cg->context['scope']}", $t->line());
}
}))->parse($ts);
$ts->reset();
return $ts;
}
private function mutate(TokenStream $ts, Ast $context, Cycle $cycle, Directives $directives, BlueContext $blueContext) : TokenStream
{
if ($this->constant) {
return $ts;
}
static $states, $parser;
$states = $states ?? new Stack();
$parser = $parser ?? traverse(token(Token::CLOAKED), consume(chain(rtoken('/^··\\w+$/')->as('expander'), either(parentheses(), braces())->as('args')))->onCommit(function (Ast $result) use($states) {
$cg = $states->current();
$expander = $result->expander;
if (\count($result->args) === 0) {
$cg->this->fail(self::E_EMPTY_EXPANDER_SLICE, (string) $expander, $expander->line());
}
$context = Map::fromKeysAndValues(['scope' => $cg->cycle->id(), 'directives' => $cg->directives, 'blueContext' => $cg->blueContext]);
$expansion = TokenStream::fromSlice($result->args);
$mutation = $cg->this->mutate(clone $expansion, $cg->context, $cg->cycle, $cg->directives, $cg->blueContext);
$mutation = $cg->this->lookupExpander($expander)($mutation, $context);
$cg->ts->inject($mutation);
}), consume(chain(rtoken('/^·\\w+|···\\w+$/')->as('label'), operator('···'), optional(parentheses()->as('delimiters')), braces()->as('expansion')))->onCommit(function (Ast $result) use($states) {
$cg = $states->current();
$context = $cg->this->lookupContext($result->label, $cg->context, self::E_UNDEFINED_EXPANSION);
$expansion = TokenStream::fromSlice($result->expansion);
$delimiters = $result->delimiters;
// normalize single context
if (array_values($context) !== $context) {
$context = [$context];
}
foreach (array_reverse($context) as $i => $subContext) {
$mutation = $cg->this->mutate(clone $expansion, (new Ast(null, $subContext))->withParent($cg->context), $cg->cycle, $cg->directives, $cg->blueContext);
if ($i !== 0) {
foreach ($delimiters as $d) {
$mutation->push($d);
}
}
$cg->ts->inject($mutation);
}
}), consume(rtoken('/^(T_\\w+·\\w+|·\\w+|···\\w+)$/')->as('label'))->onCommit(function (Ast $result) use($states) {
$cg = $states->current();
$context = $cg->this->lookupContext($result->label, $cg->context, self::E_UNDEFINED_EXPANSION);
if ($context instanceof Token) {
$cg->ts->inject(TokenStream::fromSequence($context));
} elseif (is_array($context) && \count($context)) {
$tokens = [];
array_walk_recursive($context, function (Token $token) use(&$tokens) {
$tokens[] = $token;
});
$cg->ts->inject(TokenStream::fromSlice($tokens));
}
}));
$cg = (object) ['ts' => $ts, 'context' => $context, 'directives' => $directives, 'cycle' => $cycle, 'this' => $this, 'blueContext' => $blueContext];
$states->push($cg);
$parser->parse($cg->ts);
$states->pop();
$cg->ts->reset();
if ($this->cloaked) {
traverse(consume(token(Token::CLOAKED))->onCommit(function (Ast $result) use($cg) {
$cg->ts->inject(TokenStream::fromSourceWithoutOpenTag((string) $result->token()));
}))->parse($cg->ts);
$cg->ts->reset();
}
return $cg->ts;
}
private function mutate(TokenStream $ts, Ast $context) : TokenStream
{
$cg = (object) ['ts' => clone $ts, 'context' => $context];
if ($this->unsafe && !$this->hasTag('·unsafe')) {
hygienize($cg->ts, $this->cycle->id());
}
if ($this->constant) {
return $cg->ts;
}
traverse(either(token(Token::CLOAKED), consume(chain(rtoken('/^·\\w+$/')->as('expander'), parentheses()->as('args')))->onCommit(function (Ast $result) use($cg) {
$expander = $this->lookupExpander($result->expander);
$args = [];
foreach ($result->args as $arg) {
if ($arg instanceof Token) {
$key = (string) $arg;
if (preg_match('/^·\\w+|T_\\w+·\\w+|···\\w+$/', $key)) {
$arg = $cg->context->{$key};
}
}
if (is_array($arg)) {
array_push($args, ...$arg);
} else {
$args[] = $arg;
}
}
$mutation = $expander(TokenStream::fromSlice($args), $this->cycle->id());
$cg->ts->inject($mutation);
}), consume(chain(rtoken('/^·\\w+|···\\w+$/')->as('label'), operator('···'), braces()->as('expansion')))->onCommit(function (Ast $result) use($cg) {
$index = (string) $result->label;
$context = $cg->context->{$index};
if ($context === null) {
$this->fail(self::E_EXPANSION, $index, $result->label->line(), json_encode(array_keys($cg->context->all()[0]), self::PRETTY_PRINT));
}
$expansion = TokenStream::fromSlice($result->expansion);
// normalize single context
if (array_values($context) !== $context) {
$context = [$context];
}
foreach (array_reverse($context) as $i => $subContext) {
$mutation = $this->mutate($expansion, (new Ast(null, $subContext))->withParent($cg->context));
$cg->ts->inject($mutation);
}
}), consume(rtoken('/^(T_\\w+·\\w+|·\\w+|···\\w+)$/'))->onCommit(function (Ast $result) use($cg) {
$expansion = $cg->context->{(string) $result->token()};
if ($expansion instanceof Token) {
$cg->ts->inject(TokenStream::fromSequence($expansion));
} elseif (is_array($expansion) && \count($expansion)) {
$tokens = [];
array_walk_recursive($expansion, function (Token $token) use(&$tokens) {
$tokens[] = $token;
});
$cg->ts->inject(TokenStream::fromSlice($tokens));
}
}), any()))->parse($cg->ts);
$cg->ts->reset();
if ($this->cloaked) {
traverse(either(consume(token(Token::CLOAKED))->onCommit(function (Ast $result) use($cg) {
$cg->ts->inject(TokenStream::fromSourceWithoutOpenTag((string) $result->token()));
}), any()))->parse($cg->ts);
$cg->ts->reset();
}
return $cg->ts;
}
