function yay_parse(string $source, Directives $directives = null, BlueContext $blueContext = null) : string
{
if ($gc = gc_enabled()) {
gc_disable();
}
// important optimization!
static $globalDirectives = null;
if (null === $globalDirectives) {
$globalDirectives = new ArrayObject();
}
$directives = $directives ?: new Directives();
$blueContext = $blueContext ?: new BlueContext();
$cg = (object) ['ts' => TokenStream::fromSource($source), 'directives' => $directives, 'cycle' => new Cycle($source), 'globalDirectives' => $globalDirectives, 'blueContext' => $blueContext];
foreach ($cg->globalDirectives as $d) {
$cg->directives->add($d);
}
traverse(midrule(function (TokenStream $ts) use($directives, $blueContext) {
$token = $ts->current();
tail_call:
if (null === $token) {
return;
}
// skip when something looks like a new macro to be parsed
if ('macro' === (string) $token) {
return;
}
// here we do the 'magic' to match and expand userland macros
$directives->apply($ts, $token, $blueContext);
$token = $ts->next();
goto tail_call;
}), consume(chain(token(T_STRING, 'macro')->as('declaration'), optional(repeat(rtoken('/^·\\w+$/')))->as('tags'), lookahead(token('{')), commit(chain(braces()->as('pattern'), operator('>>'), braces()->as('expansion')))->as('body'), optional(token(';'))), CONSUME_DO_TRIM)->onCommit(function (Ast $macroAst) use($cg) {
$scope = Map::fromEmpty();
$tags = Map::fromValues(array_map('strval', $macroAst->{'tags'}));
$pattern = new Pattern($macroAst->{'declaration'}->line(), $macroAst->{'body pattern'}, $tags, $scope);
$expansion = new Expansion($macroAst->{'body expansion'}, $tags, $scope);
$macro = new Macro($tags, $pattern, $expansion, $cg->cycle);
$cg->directives->add($macro);
// allocate the userland macro
// allocate the userland macro globally if it's declared as global
if ($macro->tags()->contains('·global')) {
$cg->globalDirectives[] = $macro;
}
}))->parse($cg->ts);
$expansion = (string) $cg->ts;
if ($gc) {
gc_enable();
}
return $expansion;
}
function apply(TokenStream $ts, Directives $directives, BlueContext $blueContext)
{
$from = $ts->index();
$crossover = $this->pattern->match($ts);
if (null === $crossover || $crossover instanceof Error) {
return;
}
if ($this->hasExpansion) {
$blueMacros = $this->getAllBlueMacrosFromCrossover($crossover->all(), $blueContext);
if ($this->terminal && isset($blueMacros[$this->id])) {
// already expanded
$ts->jump($from);
return;
}
$ts->unskip(...TokenStream::SKIPPABLE);
$to = $ts->index();
$ts->extract($from, $to);
$expansion = $this->expansion->expand($crossover, $this->cycle, $directives, $blueContext);
$blueMacros[$this->id] = true;
// paint blue context with tokens from expansion and disabled macros
$node = $expansion->index();
while ($node instanceof Node) {
$blueContext->addDisabledMacros($node->token, $blueMacros);
$node = $node->next;
}
$ts->inject($expansion);
} else {
$ts->unskip(...TokenStream::SKIPPABLE);
$ts->skip(T_WHITESPACE);
$to = $ts->index();
$ts->extract($from, $to);
}
$this->cycle->next();
}
/**
* Converts a token stream to a node tree.
*
* The valid names is an array where the values
* are the names that the user can use in an expression.
*
* If the variable name in the compiled PHP code must be
* different, define it as the key.
*
* For instance, ['this' => 'container'] means that the
* variable 'container' can be used in the expression
* but the compiled code will use 'this'.
*
* @param TokenStream $stream A token stream instance
* @param array $names An array of valid names
*
* @return Node A node tree
*
* @throws SyntaxError
*/
public function parse(TokenStream $stream, $names = array())
{
$this->stream = $stream;
$this->names = $names;
$node = $this->parseExpression();
if (!$stream->isEOF()) {
throw new SyntaxError(sprintf('Unexpected token "%s" of value "%s"', $stream->current->type, $stream->current->value), $stream->current->cursor);
}
return $node;
}
function hygienize(TokenStream $ts, string $scope) : TokenStream
{
$ts->reset();
traverse(either(chain(token(T_STRING, '·unsafe'), parentheses()), either(token(T_VARIABLE)->as('target'), chain(identifier()->as('target'), token(':')), chain(token(T_GOTO), identifier()->as('target')))->onCommit(function (Ast $result) use($scope) {
(function () use($scope) {
if ((string) $this !== '$this') {
$this->value = (string) $this . '·' . $scope;
}
})->call($result->target);
}), any()))->parse($ts);
$ts->reset();
return $ts;
}
function tokenize($source)
{
$result = new TokenStream();
$pos = 0;
$matches = array();
preg_match_all($this->pattern, $source, $matches, PREG_SET_ORDER);
foreach ($matches as $match) {
if ($match[1]) {
$result->feed('text', $match[1], $pos);
}
$tagpos = $pos + strlen($match[1]);
if ($match[2]) {
$result->feed('block', trim($match[2]), $tagpos);
} elseif ($match[3]) {
$result->feed('variable', trim($match[3]), $tagpos);
} elseif ($match[4]) {
$result->feed('comment', trim($match[4]), $tagpos);
}
$pos += strlen($match[0]);
}
if ($pos < strlen($source)) {
$result->feed('text', substr($source, $pos), $pos);
}
$result->close();
return $result;
}
function tokenize($source)
{
$result = new TokenStream();
$pos = 0;
$matches = array();
preg_match_all($this->pattern, $source, $matches, PREG_OFFSET_CAPTURE | PREG_SET_ORDER);
if (preg_last_error() == PREG_BACKTRACK_LIMIT_ERROR) {
throw new TemplateSyntaxError('Template too large, PCRE backtrack limit reached');
}
foreach ($matches as $match) {
if (!isset($match["match"])) {
throw new TemplateSyntaxError('Template error');
}
$tagpos = $match["match"][1];
if ($tagpos > 0) {
$text = substr($source, $pos, $tagpos - $pos);
$result->feed('text', $text, $pos);
}
if (isset($match["block"]) && $match["block"][1] != -1) {
$result->feed('block', trim($match["block"][0]), $tagpos);
} elseif (isset($match["variable"]) && $match["variable"][1] != -1) {
$result->feed('variable', trim($match["variable"][0]), $tagpos);
} elseif (iset($match["comment"]) && $match["comment"][1] != -1) {
$result->feed('comment', trim($match["comment"][0]), $tagpos);
}
$pos = $tagpos + strlen($match["match"][0]);
}
if ($pos < strlen($source)) {
$result->feed('text', substr($source, $pos), $pos);
}
$result->close();
return $result;
}
/**
*
* @param TokenStream $stream
* @param int $indention
* @return TokenStreamBuilder
*/
public function addStream(TokenStream $stream, $indention = 1)
{
if ($indention > 0) {
$stream->indent($indention);
}
$this->stream->append($stream);
return $this;
}
private function compile(array $tokens)
{
$cg = (object) ['ts' => TokenStream::fromSlice($tokens), 'parsers' => []];
traverse(rtoken('/^(T_\\w+)·(\\w+)$/')->onCommit(function (Ast $result) use($cg) {
$token = $result->token();
$id = $this->lookupCapture($token);
$type = $this->lookupTokenType($token);
$cg->parsers[] = token($type)->as($id);
}), ($parser = chain(rtoken('/^·\\w+$/')->as('type'), token('('), optional(ls(either(future($parser)->as('parser'), chain(token(T_FUNCTION), parentheses()->as('args'), braces()->as('body'))->as('function'), string()->as('string'), rtoken('/^T_\\w+·\\w+$/')->as('token'), rtoken('/^T_\\w+$/')->as('constant'), rtoken('/^·this$/')->as('this'), label()->as('label'))->as('parser'), token(',')))->as('args'), commit(token(')')), optional(rtoken('/^·\\w+$/')->as('label'), null)))->onCommit(function (Ast $result) use($cg) {
$cg->parsers[] = $this->compileParser($result->type, $result->args, $result->label);
}), $this->layer('{', '}', braces(), $cg), $this->layer('[', ']', brackets(), $cg), $this->layer('(', ')', parentheses(), $cg), rtoken('/^···(\\w+)$/')->onCommit(function (Ast $result) use($cg) {
$id = $this->lookupCapture($result->token());
$cg->parsers[] = layer()->as($id);
}), token(T_STRING, '·')->onCommit(function (Ast $result) use($cg) {
$offset = \count($cg->parsers);
if (0 !== $this->dominance || 0 === $offset) {
$this->fail(self::E_BAD_DOMINANCE, $offset, $result->token()->line());
}
$this->dominance = $offset;
}), rtoken('/·/')->onCommit(function (Ast $result) {
$token = $result->token();
$this->fail(self::E_BAD_CAPTURE, $token, $token->line());
}), any()->onCommit(function (Ast $result) use($cg) {
$cg->parsers[] = token($result->token());
}))->parse($cg->ts);
// check if macro dominance '·' is last token
if ($this->dominance === \count($cg->parsers)) {
$this->fail(self::E_BAD_DOMINANCE, $this->dominance, $cg->ts->last()->line());
}
$this->specificity = \count($cg->parsers);
if ($this->specificity > 1) {
if (0 === $this->dominance) {
$pattern = chain(...$cg->parsers);
} else {
/*
dominat macros are partially wrapped in commit()s and dominance
is the offset used as the 'event horizon' point... once the entry
point is matched, there is no way back and a parser error arises
*/
$prefix = array_slice($cg->parsers, 0, $this->dominance);
$suffix = array_slice($cg->parsers, $this->dominance);
$pattern = chain(...array_merge($prefix, array_map(commit::class, $suffix)));
}
} else {
/*
micro optimization to save one function call for every token on the subject
token stream whenever the macro pattern consists of a single parser
*/
$pattern = $cg->parsers[0];
}
return $pattern;
}
<?php
/*
This script will remove all whitespace and comments from itself
*/
error_reporting(E_ALL | E_STRICT);
/*
TokenStream implementation
*/
require '../src/TokenStream.php';
$tokens = new TokenStream(file_get_contents(__FILE__));
$i = 0;
while ($i = $tokens->find($i, array(T_WHITESPACE, T_COMMENT, T_DOC_COMMENT))) {
// if the whitespace separates two variables/labels, e.g. new TokenStream or $tokens as $token
// only convert token to a space character
if (isset($tokens[$i - 1], $tokens[$i + 1]) && ($tokens[$i - 1]->is(T_STRING, T_VARIABLE) || ctype_alpha($tokens[$i - 1]->content)) && ($tokens[$i + 1]->is(T_STRING) || ctype_alpha($tokens[$i + 1]->content))) {
$tokens[$i] = new Token(T_WHITESPACE, ' ');
} else {
unset($tokens[$i--]);
// $i-- because this token ought to be rechecked
}
}
echo '<pre>', htmlspecialchars($tokens), '</pre>';
/*
"native" implementation
*/
$tokens = token_get_all(file_get_contents(__FILE__));
for ($i = 0; $i < count($tokens); ++$i) {
if (is_array($tokens[$i]) && in_array($tokens[$i][0], array(T_WHITESPACE, T_COMMENT, T_DOC_COMMENT))) {
// if the whitespace separates two variables/labels, e.g. new TokenStream or $tokens as $token
// only convert token to a space character
/**
* Parses an attribute from a selector contained in $stream and returns
* the resulting AttribNode object.
*
* @throws SyntaxError When encountered unexpected selector
*
* @param Node\NodeInterface $selector The selector object whose attribute
* is to be parsed.
* @param TokenStream $stream The container token stream.
*
* @return Node\AttribNode
*/
protected function parseAttrib($selector, $stream)
{
$attrib = $stream->next();
if ($stream->peek() == '|') {
$namespace = $attrib;
$stream->next();
$attrib = $stream->next();
} else {
$namespace = '*';
}
if ($stream->peek() == ']') {
return new Node\AttribNode($selector, $namespace, $attrib, 'exists', null);
}
$op = $stream->next();
if (!in_array($op, array('^=', '$=', '*=', '=', '~=', '|=', '!='))) {
throw new SyntaxError(sprintf("Operator expected, got '%s'", $op));
}
$value = $stream->next();
if (!$value->isType('Symbol') && !$value->isType('String')) {
throw new SyntaxError(sprintf("Expected string or symbol, got '%s'", $value));
}
return new Node\AttribNode($selector, $namespace, $attrib, $op, $value);
}
function processTokenStream(TokenStream $tokenStream, ConverterStateMachine $stateMachine, $originalFilename)
{
$name = '';
$value = '';
$stateMachine->currentTokenStream = $tokenStream;
while ($tokenStream->hasMoreTokens() == TRUE) {
$tokenStream->next($name, $value);
$count = 0;
$parsedToken = $stateMachine->parseToken($name, $value, $count);
//TODO - both of these should be somewhere more logical.
if ($name == 'T_CONSTANT_ENCAPSED_STRING') {
$value = convertMultiLineString($value);
}
if ($name == 'T_ENCAPSED_AND_WHITESPACE') {
$parsedToken = convertMultiLineString($parsedToken);
}
$stateMachine->accountForOpenBrackets($name);
$stateMachine->accountForQuotes($name);
$stateMachine->scopePreStateMagic($name, $value);
do {
$reprocess = $stateMachine->processToken($name, $value, $parsedToken);
if ($count > 5) {
throw new \Exception("Stuck converting same token.");
}
$count++;
} while ($reprocess == TRUE);
$stateMachine->accountForCloseBrackets($name);
$stateMachine->scopePostStateMagic($name, $value);
if ($name == 'T_VARIABLE') {
//If there's a token that needs to be inserted e.g. 'var'
if ($stateMachine->insertToken != FALSE) {
$stateMachine->addJS($stateMachine->insertToken);
$stateMachine->insertToken = FALSE;
}
}
if (FALSE) {
$requiredFile = $stateMachine->getRequiredFile();
if ($requiredFile != NULL) {
//echo "Figure out where $requiredFile is from original file path $originalFilename";
//TraitInclude.php' is from original file path TraitExample.php
$pathParts = pathinfo($originalFilename);
$requireFilePath = $pathParts['dirname'] . '/' . $requiredFile;
//$requireFilePath = realpath($requireFilePath);
if (PHPToJavascript::$TRACE == TRUE) {
echo "Including file [{$requiredFile}] on path [{$requireFilePath}].";
}
$code = file_get_contents($requireFilePath);
if ($code === FALSE) {
throw new \Exception("Could not open file [{$requiredFile}] on path [{$requireFilePath}].");
}
$requireTokenStream = new TokenStream($code);
processTokenStream($requireTokenStream, $stateMachine, $originalFilename);
$stateMachine->addJS("\n//End of require\n");
//TODO Add a new state to tidy up semi-colon after include
}
}
}
}
private function mutate(TokenStream $ts, Ast $context, Cycle $cycle, Directives $directives, BlueContext $blueContext) : TokenStream
{
if ($this->constant) {
return $ts;
}
static $states, $parser;
$states = $states ?? new Stack();
$parser = $parser ?? traverse(token(Token::CLOAKED), consume(chain(rtoken('/^··\\w+$/')->as('expander'), either(parentheses(), braces())->as('args')))->onCommit(function (Ast $result) use($states) {
$cg = $states->current();
$expander = $result->expander;
if (\count($result->args) === 0) {
$cg->this->fail(self::E_EMPTY_EXPANDER_SLICE, (string) $expander, $expander->line());
}
$context = Map::fromKeysAndValues(['scope' => $cg->cycle->id(), 'directives' => $cg->directives, 'blueContext' => $cg->blueContext]);
$expansion = TokenStream::fromSlice($result->args);
$mutation = $cg->this->mutate(clone $expansion, $cg->context, $cg->cycle, $cg->directives, $cg->blueContext);
$mutation = $cg->this->lookupExpander($expander)($mutation, $context);
$cg->ts->inject($mutation);
}), consume(chain(rtoken('/^·\\w+|···\\w+$/')->as('label'), operator('···'), optional(parentheses()->as('delimiters')), braces()->as('expansion')))->onCommit(function (Ast $result) use($states) {
$cg = $states->current();
$context = $cg->this->lookupContext($result->label, $cg->context, self::E_UNDEFINED_EXPANSION);
$expansion = TokenStream::fromSlice($result->expansion);
$delimiters = $result->delimiters;
// normalize single context
if (array_values($context) !== $context) {
$context = [$context];
}
foreach (array_reverse($context) as $i => $subContext) {
$mutation = $cg->this->mutate(clone $expansion, (new Ast(null, $subContext))->withParent($cg->context), $cg->cycle, $cg->directives, $cg->blueContext);
if ($i !== 0) {
foreach ($delimiters as $d) {
$mutation->push($d);
}
}
$cg->ts->inject($mutation);
}
}), consume(rtoken('/^(T_\\w+·\\w+|·\\w+|···\\w+)$/')->as('label'))->onCommit(function (Ast $result) use($states) {
$cg = $states->current();
$context = $cg->this->lookupContext($result->label, $cg->context, self::E_UNDEFINED_EXPANSION);
if ($context instanceof Token) {
$cg->ts->inject(TokenStream::fromSequence($context));
} elseif (is_array($context) && \count($context)) {
$tokens = [];
array_walk_recursive($context, function (Token $token) use(&$tokens) {
$tokens[] = $token;
});
$cg->ts->inject(TokenStream::fromSlice($tokens));
}
}));
$cg = (object) ['ts' => $ts, 'context' => $context, 'directives' => $directives, 'cycle' => $cycle, 'this' => $this, 'blueContext' => $blueContext];
$states->push($cg);
$parser->parse($cg->ts);
$states->pop();
$cg->ts->reset();
if ($this->cloaked) {
traverse(consume(token(Token::CLOAKED))->onCommit(function (Ast $result) use($cg) {
$cg->ts->inject(TokenStream::fromSourceWithoutOpenTag((string) $result->token()));
}))->parse($cg->ts);
$cg->ts->reset();
}
return $cg->ts;
}
test($tokenStream->findEOS(6), 12, 'find eos tag');
test($tokenStream->find(0, T_OPEN_TAG), false, 'find from already matchind token');
test($tokenStream->find($max, T_EXIT, true), 2, 'find token backwards');
test($tokenStream->find(0, T_OPEN_TAG, true), false, 'find from first index backwards');
test($tokenStream->findEOS($max, true), 6, 'find eos semicolon');
// 0 123 4567890
$hacker = new TokenStream('<?php A(function(){;});');
test($hacker->findEOS(2), 10, 'find EOS skipping lambda');
test($hacker->findEOS(9, true), 0, 'find EOS skipping lambda backwards');
// 0 123
$hacker = new TokenStream('<?php {} ');
test($hacker->findEOS(3, true), 2, 'find closing bracket as EOS backwards');
});
group('skip', function () use($tokenStream, $max) {
test($tokenStream->skip(0, T_WHITESPACE), 2, 'skip token');
test($tokenStream->skip(0, array(T_WHITESPACE, T_EXIT)), 3, 'skip tokens');
test($tokenStream->skip($max, array(T_WHITESPACE, T_CONSTANT_ENCAPSED_STRING), true), 8, 'skip tokens backwards');
test($tokenStream->skipWhitespace(0), 2, 'skip whitespace');
});
// 0 12 34 5678901 234 56 789
$tokenStream = new TokenStream('<?php (hi,hi,(),((hi),hi)) (()');
group('complementaryBracket', function () use($tokenStream) {
test($tokenStream->complementaryBracket(1), 16, 'find forward');
test($tokenStream->complementaryBracket(15), 9, 'find backwards');
testException(function () use($tokenStream) {
$tokenStream->complementaryBracket(17);
}, 'TokenException', 'brackets not matching');
testException(function () use($tokenStream) {
$tokenStream->complementaryBracket(2);
}, 'TokenException', 'not a bracket');
});
function getPreviousNonWhitespaceToken(&$name, &$value)
{
return $this->currentTokenStream->getPreviousNonWhitespaceToken($name, $value);
}
function yay_parse(string $source, string $salt) : string
{
start:
if ($gc = gc_enabled()) {
gc_disable();
}
$ts = TokenStream::fromSource($source);
$directives = new Directives();
$cycle = new Cycle($salt);
declaration:
$from = $ts->index();
if (!($token = $ts->current())) {
goto end;
}
if ($token->contains('macro')) {
$declaration = $token;
$ts->next();
goto tags;
}
goto any;
tags:
$tags = [];
while (($token = $ts->current()) && preg_match('/^·\\w+$/', (string) $token)) {
$tags[] = $token;
$ts->next();
}
pattern:
$index = $ts->index();
$pattern = [];
$level = 1;
if (!($token = $ts->current()) || !$token->is('{')) {
goto any;
}
$ts->next();
while (($token = $ts->current()) && ($level += LAYER_DELIMITERS[$token->type()] ?? 0)) {
$pattern[] = $token;
$token = $ts->step();
}
if (!$token || !$token->is('}')) {
$ts->jump($index);
(new Error(new Expected(new Token('}')), $ts->current(), $ts->last()))->halt();
}
$ts->next();
__:
// >>
$index = $ts->index();
$operator = '>>';
$max = strlen($operator);
$buffer = '';
while (mb_strlen($buffer) <= $max && ($token = $ts->current())) {
$buffer .= (string) $token;
$ts->step();
if ($buffer === $operator) {
$ts->skip(T_WHITESPACE);
goto expansion;
}
}
$ts->jump($index);
(new Error(new Expected(new Token(token::OPERATOR, $operator)), $ts->current(), $ts->last()))->halt();
expansion:
$index = $ts->index();
$expansion = [];
$level = 1;
if (!($token = $ts->current()) || !$token->is('{')) {
$ts->jump($index);
(new Error(new Expected(new Token('}')), $ts->current(), $ts->last()))->halt();
}
$ts->next();
while (($token = $ts->current()) && ($level += LAYER_DELIMITERS[$token->type()] ?? 0)) {
$expansion[] = $token;
$token = $ts->step();
}
if (!$token || !$token->is('}')) {
$ts->jump($index);
(new Error(new Expected(new Token('}')), $ts->current(), $ts->last()))->halt();
}
// optional ';'
$token = $ts->next();
if ($token->is(';')) {
$ts->next();
}
// cleanup
$ts->unskip(...TokenStream::SKIPPABLE);
$ts->skip(T_WHITESPACE);
$ts->extract($from, $ts->index());
$directives->add(new Macro($declaration->line(), $tags, $pattern, $expansion, $cycle));
goto declaration;
any:
if ($token) {
$directives->apply($ts);
$ts->next();
}
goto declaration;
end:
$expansion = (string) $ts;
if ($gc) {
gc_enable();
}
return $expansion;
}
function D(TokenStream $ts)
{
$cur = $ts->GetCurrentKind();
if ($cur == Kind::digit_0 or $cur == Kind::digit_1 or $cur == Kind::digit_2 or $cur == Kind::digit_3 or $cur == Kind::digit_4 or $cur == Kind::digit_5 or $cur == Kind::digit_6 or $cur == Kind::digit_7 or $cur == Kind::digit_8 or $cur == Kind::digit_9) {
$ts->MoveNext();
return new OutValue($cur, $cur);
} else {
throw new Exception("Input is incorrect.");
}
}
/**
* PHP Internal functions
* @string $int_functions is buildin function to get all php core functions
* @global $this->_classes included all classes in files
* @global $this->_fullclasses is @array included each class and all functions related to it
* @global $this->FuncArray is @array included all functions and class if requested
* @global $this->ConstArray is @array included all constants definded in file
* @global $this->VarArray is @array included all vars definded in file
* @return void
*/
function PHPFetchContent($contents, $analyze = false, $file = false)
{
//$contents = file_get_contents('dbconnector.php');
$tokens = new TokenStream($contents);
$int_functions = get_defined_functions();
$this->fetchExcluded($file);
$i = 0;
//echo '<pre>';
//print_r($tokens->debugDump());
//echo '</pre>';
while ($i = $tokens->skipWhitespace($i)) {
if ($tokens[$i]->is(T_CLASS)) {
//extract class and assign it to class object
$this->class = $tokens[$i + 2]->content;
if (input::get('ReplaceClasses')) {
//extract class and assign it to class object
$this->class = $tokens[$i + 2]->content;
if (!in_array($this->class, $this->_classes)) {
$this->_classes[] = $this->class;
}
//obfuscate the class object with prefix "C"
$class_obfuc = "C" . substr(md5($this->class), 0, 8);
//generate classes and encoded
if (!in_array($this->class, $this->ClassArray)) {
//if process is not practical analyze, obfuscate it
//if(!$analyze)
$this->ClassArray[$this->class] = $class_obfuc;
}
//generate class name and replace it
$tokens[$i + 2]->content = $class_obfuc;
}
//if(!isset($this->_fullclasses['classes']))
// $this->_fullclasses['classes']['NONE_OOP'] = 'on';
} elseif ($tokens->findEOS($i, false) && $tokens[$i]->is(T_PUBLIC, T_PRIVATE, T_PROTECTED, T_VAR, T_STATIC)) {
/**
* check if after the T_* is not 'static' object
* check if after the T_* is not 'function' object
* private static $_singleton;
*/
if (!$tokens[$i + 2]->is(T_STATIC) && !$tokens[$i + 2]->is(T_FUNCTION)) {
$defineded_vars = $tokens[$i + 2]->content;
//so ? assign the existent vars to check it later!
if (!in_array($defineded_vars, $this->existent_vars)) {
$this->existent_vars[substr($tokens[$i + 2]->content, 1)] = $defineded_vars;
}
}
//echo '<pre>';
//print_r($this->existent_vars);
//echo '</pre>';
} elseif ($tokens[$i]->is(T_NEW)) {
/**
* search where class is used and replace it
* check if class in our classes
* fixme: fix the object it added to class
*/
if ($tokens[$i + 2]->is(T_STRING) and in_array($tokens[$i + 2]->content, array_flip($this->ClassArray))) {
$set_in = array_search($tokens[$i + 2]->content, $this->_fullclasses['classes']);
$tokens[$i + 2]->content = $this->ClassArray[$tokens[$i + 2]->content];
//if(isset($this->_fullclasses['classes'][$set_in]))
//$this->_fullclasses['classes'][$set_in] = $tokens[$i-4]->content;
}
} elseif ($tokens[$i]->is(T_FUNCTION)) {
if (input::get('ReplaceFunctions')) {
//extract function and assign it to function object
$this->function = $tokens[$i + 2]->content;
//check if the function not in exclude list
if (!in_array($this->function, $this->UdExcFuncArray)) {
//check if the function object not defined in functions list
if (!isset($this->FuncArray[$this->function])) {
/**
* Array ( [MAP_SETTINGS] => F8018a28e [getAutocomplete] => F48b9a789 )
* generate function name and push to FuncArray
**/
$this->FuncArray[$this->function] = "F" . substr(md5($this->function), 0, 8);
}
// it's Absolutely an function !
if ($tokens[$i + 3]->content == '(') {
// generate functions tree under the main class
if ($this->function and $this->class) {
//check if the function not defined
if (!isset($this->_fullclasses['classes'][$this->class]['functions'][$this->function])) {
//assign it to class
$this->_fullclasses['classes'][$this->class]['functions'][$this->function] = $this->function;
}
}
//obfuscate the function object with prefix "F"
$tokens[$i + 2]->content = "F" . substr(md5($this->function), 0, 8);
}
}
//extends inside functions for ...
//$extends = $tokens->find($i,T_VARIABLE);
//Debugbar::info($tokens[$extends+2]->content);
/**
* extends search if the function is used inside the current function and replace it
* @example
* $this->function
* fixme: need to check if it real function not sample var
if($tokens[$extends+2]->is(T_STRING) and in_array($tokens[$extends+2]->content, array_flip($this->FuncArray)) ){
## set the function name
$this->function = $tokens[$extends+2]->content;
if(isset($this->_fullclasses['classes'][$this->class]['functions'][$this->function])){
$tokens[$extends+2]->content = $this->FuncArray[$this->function];
}else
$tokens[$extends+2]->content = $this->FuncArray[$this->function];
}
*/
}
} elseif ($tokens[$i]->is(T_VARIABLE)) {
if (input::get('ReplaceVariables')) {
//clear $ from var
$VarName = substr($tokens[$i]->content, 1);
/**
* extract the #variable and obfuscate it
* Absolutely skip $this object
* check again in exclude variables array list
* TODO: Check if there duplicated vars and it is an objects
* $tokens[$i+1]->type != T_OBJECT_OPERATOR '->'
*/
if ($tokens[$i]->content != '$this' && !in_array($VarName, $this->UdExcVarArray)) {
//check if the var no defineded in variables array list
//if process is not practical analyze, obfuscate it
if (!isset($this->VarArray[$VarName])) {
//obfuscate the variable string with prefix "V"
$this->VarArray[$VarName] = 'V' . substr(md5(uniqid($VarName, true)), 0, mt_rand(5, 12));
}
$tokens[$i]->content = "\${$this->VarArray[$VarName]}";
//fixme: #exclude the vars in existent_vars from vars
$this->_fullclasses['classes'][$this->class]['vars'][$VarName] = $VarName;
//$tokens[$i]->content = '${\''.$this->encode_string($this->VarArray[$VarName]).'\'}';
}
/**
* Search for objects and push founded to objects array
*/
if ($tokens[$i]->content != '$this' && $tokens[$i + 1]->type == T_OBJECT_OPERATOR) {
if (!empty($VarName) && !in_array($VarName, $this->ObjectArray)) {
array_push($this->ObjectArray, $VarName);
}
}
/**
* search for vars after $this need to replace
* $var or function ?
* $this->object
*/
if ($tokens[$i]->content == '$this' && $tokens[$i + 1]->type == T_OBJECT_OPERATOR) {
if ($tokens[$i + 2]->is(T_STRING)) {
//check if the string after $this in existent vars array list
if (array_key_exists($tokens[$i + 2]->content, $this->existent_vars)) {
//make sure it's not an function have a duplicate var name
if ($tokens[$i + 3]->content != '(') {
$this->_fullclasses['classes'][$this->class]['existent_vars'][$tokens[$i + 2]->content] = $tokens[$i + 2]->content;
//if process is not practical analyze, obfuscate it
//if(!$analyze)
if (!in_array($tokens[$i + 2]->content, $this->UdExcVarArray)) {
$tokens[$i + 2]->content = $this->VarArray[$tokens[$i + 2]->content];
}
}
}
//so ? maybe it's a function okay check it
if ($tokens[$i + 3]->content == '(') {
if (in_array($tokens[$i + 2]->content, array_flip($this->FuncArray))) {
//hmmm okay f**k it too!
//if process is not practical analyze, obfuscate it
//if(!$analyze)
$tokens[$i + 2]->content = $this->FuncArray[$tokens[$i + 2]->content];
}
}
}
}
}
} elseif ($tokens[$i]->is(T_CONSTANT_ENCAPSED_STRING)) {
//standard PHP ENV variables that should be replaced
//if(in_array(substr($tokens[$i-2]->content, 1),$this->StdExcVarArray)){
//check if the this string start with single quote
//if (preg_match('/^\'(.*)\'$/', $tokens[$i]->content)) {
if (preg_match('/\'.*(.*)\'/Uis', $tokens[$i]->content) && input::get('ReplaceEncode')) {
$clean_quote = trim($tokens[$i]->content, "'");
//endcode it and add double quote
$tokens[$i]->content = MagicalHelpers::random_encode("\"{$clean_quote}\"");
}
//}else{
//if (!preg_match('/^\'(.*)\'$/', $tokens[$i]->content))
// $tokens[$i]->content = MagicalHelpers::random_encode($tokens[$i]->content);
//}
} elseif ($tokens[$i]->is(T_STRING)) {
/**
* wide check ..
* static method CLASS::FUNCTION()
* OOP method CLASS->FUNCTION()
*/
if ($tokens[$i - 1]->is(T_DOUBLE_COLON) || $tokens[$i - 1]->is(T_OBJECT_OPERATOR)) {
//check if it's an function
if ($tokens[$i + 1]->content == '(') {
//search if it definded in functions array list
if (in_array($tokens[$i]->content, array_flip($this->FuncArray))) {
$class = $tokens[$i - 2]->content;
$function = $tokens[$i]->content;
//check if this class exists in class array list
if (in_array($class, $this->_fullclasses['classes'])) {
$found = false;
//deep check
array_walk_recursive($this->_fullclasses['classes'], function ($v, $k) use(&$found) {
$found |= $k == $this->class;
});
if ($found) {
##TODO: add check if class obfuse is required
if (in_array($class, $this->_fullclasses['classes'])) {
//if process is not practical analyze, obfuscate it
if (isset($this->ClassArray[$class])) {
$tokens[$i - 2]->content = $this->ClassArray[$class];
}
}
}
}
if (isset($function, $this->_fullclasses['classes']["{$class}"])) {
//check if this function assigned into this class
if (in_array($function, $this->_fullclasses['classes']["{$class}"]['functions'])) {
//if process is not practical analyze, obfuscate it
//if(!$analyze)
$tokens[$i]->content = $this->FuncArray[$tokens[$i]->content];
}
}
}
}
}
/**
* search where the function is used and replace it
* support none OOP, inline function
*/
if ($tokens[$i - 1]->type != T_OBJECT_OPERATOR && $tokens[$i - 1]->type != T_DOUBLE_COLON) {
$string_name = $tokens[$i]->content;
if (!in_array($string_name, $int_functions['internal']) and in_array($string_name, array_flip($this->FuncArray))) {
$this->_fullclasses['classes']['NONE_OOP']['functions'][$string_name] = $string_name;
//if process is not practical analyze, obfuscate it
//if(!$analyze)
$tokens[$i]->content = $this->FuncArray[$string_name];
}
}
}
}
$contents = $tokens;
if (Input::has('Analyze')) {
//echo '<pre>';
//print_r(array_merge_recursive($this->_fullclasses,array('classes_exists' => $this->_classes)));
//echo '</pre>';
if (!empty($this->_classes)) {
return array_merge_recursive($this->_fullclasses, array('classes_exists' => $this->_classes));
} else {
return $this->_fullclasses;
}
} else {
return $contents;
}
}
function expand(TokenStream $ts, Context $context) : TokenStream
{
$ts = TokenStream::fromSource(yay_parse('<?php ' . (string) $ts, $context->get('directives'), $context->get('blueContext')));
$ts->shift();
return $ts;
}
function testastNestedAst()
{
$ts = TokenStream::fromSource('<?php
interface Foo
{
public abstract function foo();
public abstract static function bar();
function baz();
}
');
$modifier = either(token(T_PUBLIC), token(T_STATIC), token(T_PRIVATE));
$modifiers = optional(either(chain($modifier, $modifier), $modifier));
$ast = chain(token(T_OPEN_TAG), chain(token(T_INTERFACE), token(T_STRING)->as('name'), token('{'), optional(repeat(chain(optional(either(repeat(either(token(T_PUBLIC)->as('public'), token(T_STATIC)->as('static'), token(T_ABSTRACT)->as('abstract'))), always(new Token(T_PUBLIC, 'public'))->as('public')))->as('is'), token(T_FUNCTION), token(T_STRING)->as('name'), token('('), token(')'), token(';')), token('}')))->as('methods'), token('}'))->as('interface'))->parse($ts);
$this->assertEquals('Foo', (string) $ast->{'interface name'});
$this->assertEquals('foo', (string) $ast->{'interface methods 0 name'});
$this->assertEquals('public', (string) $ast->{'interface methods 0 is public'});
$this->assertEquals('abstract', (string) $ast->{'interface methods 0 is abstract'});
$this->assertEmpty($ast->{'interface methods 0 is static'});
$this->assertEquals('bar', (string) $ast->{'interface methods 1 name'});
$this->assertEquals('public', (string) $ast->{'interface methods 1 is public'});
$this->assertEquals('abstract', (string) $ast->{'interface methods 1 is abstract'});
$this->assertEquals('static', (string) $ast->{'interface methods 1 is static'});
$this->assertEquals('baz', (string) $ast->{'interface methods 2 name'});
$this->assertEquals('public', (string) $ast->{'interface methods 2 is public'});
$this->assertNull($ast->{'interface methods 2 is abstract'});
$this->assertNull($ast->{'interface methods 2 is static'});
}
<?php
require './test.php';
require '../src/TokenStream.php';
$tokenStream = new TokenStream("<?php\ndie();");
group('__toString', function () use($tokenStream) {
test((string) $tokenStream, "<?php\ndie();", '(string) $tokenStream');
});
echo '<p>The debugDump method is hard to test. But try this:
If you like the following output, everything\'s okay. If not, test failed ;)</p>';
$tokenStream = new TokenStream(file_get_contents(__FILE__));
$tokenStream->debugDump(true, true, false);
public function append(TokenStream $stream)
{
$this->tokens = array_merge($this->tokens, $stream->getTokens());
return $this;
}
<?php
require './test.php';
require '../src/TokenStream.php';
$tokenStream = new TokenStream("<?php\ndie();");
group('get', function () use($tokenStream) {
test(streamEqual($tokenStream->get(1, 2), array(new Token(T_EXIT, 'die'), new Token(T_OPEN_ROUND, '('))), true, 'get(int, int)');
});
group('extract', function () use($tokenStream) {
$tokenStream = clone $tokenStream;
test(streamEqual($tokenStream->extract(1, 2), array(new Token(T_EXIT, 'die'), new Token(T_OPEN_ROUND, '('))), true, 'return value of extract(int, int)');
test(streamEqual($tokenStream->get(0, count($tokenStream) - 1), array(new Token(T_OPEN_TAG, "<?php\n"), new Token(T_CLOSE_ROUND, ')'), new Token(T_SEMICOLON, ';'))), true, 'stream change of extract(int, int)');
test(tokenEqual($tokenStream->extract(2), new Token(T_SEMICOLON, ';')), true, 'return value of extract(int)');
test(streamEqual($tokenStream->get(0, count($tokenStream) - 1), array(new Token(T_OPEN_TAG, "<?php\n"), new Token(T_CLOSE_ROUND, ')'))), true, 'stream change of extract(int)');
});
group('append', function () use($tokenStream) {
$tokenStream = clone $tokenStream;
$tokenStream->append(array(new Token(T_WHITESPACE, "\n"), '{', '}', array(';')));
test((string) $tokenStream, "<?php\ndie();\n{};", 'append token array');
$tokenStream->append(';');
test((string) $tokenStream, "<?php\ndie();\n{};;", 'append single token');
});
group('insert', function () use($tokenStream) {
$tokenStream = clone $tokenStream;
$tokenStream->insert(1, array('{', '}', array(';', new Token(T_WHITESPACE, "\n"))));
test((string) $tokenStream, "<?php\n{};\ndie();", 'insert token array');
$tokenStream->insert(4, ';');
test((string) $tokenStream, "<?php\n{};;\ndie();", 'insert single token');
});
private function parseIdentifier(TokenStream $tokens)
{
$cur = $tokens->current();
$node = new Node\IdentifierNode($cur->getValue(), $cur);
$tokens->next();
return $node;
}
protected function parseDictKey(TokenStream $stream)
{
$stream->expect(array(Tokens::T_IDENTIFIER, Tokens::T_STRING));
$token = $stream->current();
switch ($token->name) {
case Tokens::T_STRING:
$key = $this->cleanString($token->value);
break;
case Tokens::T_IDENTIFIER:
default:
$key = $token->value;
break;
}
$stream->next();
return $key;
}
function testPush()
{
$ts = TokenStream::fromSource("<?php 1 2 3 ");
$ts->push(new Token(T_LNUMBER, '4'));
$ts->push(new Token(T_WHITESPACE, ' '));
$ts->push(new Token(T_LNUMBER, '5'));
$ts->push(new Token(T_WHITESPACE, ' '));
$this->assertEquals('<?php 1 2 3 4 5 ', (string) $ts);
$ts = TokenStream::fromEmpty();
$ts->push(new Token(T_STRING, 'A'));
$ts->push(new Token(T_WHITESPACE, ' '));
$ts->push(new Token(T_STRING, 'B'));
$ts->push(new Token(T_WHITESPACE, ' '));
$this->assertEquals('A B ', (string) $ts);
$ts->extract($ts->index(), $ts->index()->next->next->next);
$this->assertEquals(' ', (string) $ts);
$ts->push(new Token(T_STRING, 'C'));
$this->assertEquals(' C', (string) $ts);
}
function __construct(FileAccess $path, TokenStream $token_stream)
{
$this->path = $path;
$this->token_stream = $token_stream;
$this->hash = $token_stream->getHash();
}
private function mutate(TokenStream $ts, Ast $context) : TokenStream
{
$cg = (object) ['ts' => clone $ts, 'context' => $context];
if ($this->unsafe && !$this->hasTag('·unsafe')) {
hygienize($cg->ts, $this->cycle->id());
}
if ($this->constant) {
return $cg->ts;
}
traverse(either(token(Token::CLOAKED), consume(chain(rtoken('/^·\\w+$/')->as('expander'), parentheses()->as('args')))->onCommit(function (Ast $result) use($cg) {
$expander = $this->lookupExpander($result->expander);
$args = [];
foreach ($result->args as $arg) {
if ($arg instanceof Token) {
$key = (string) $arg;
if (preg_match('/^·\\w+|T_\\w+·\\w+|···\\w+$/', $key)) {
$arg = $cg->context->{$key};
}
}
if (is_array($arg)) {
array_push($args, ...$arg);
} else {
$args[] = $arg;
}
}
$mutation = $expander(TokenStream::fromSlice($args), $this->cycle->id());
$cg->ts->inject($mutation);
}), consume(chain(rtoken('/^·\\w+|···\\w+$/')->as('label'), operator('···'), braces()->as('expansion')))->onCommit(function (Ast $result) use($cg) {
$index = (string) $result->label;
$context = $cg->context->{$index};
if ($context === null) {
$this->fail(self::E_EXPANSION, $index, $result->label->line(), json_encode(array_keys($cg->context->all()[0]), self::PRETTY_PRINT));
}
$expansion = TokenStream::fromSlice($result->expansion);
// normalize single context
if (array_values($context) !== $context) {
$context = [$context];
}
foreach (array_reverse($context) as $i => $subContext) {
$mutation = $this->mutate($expansion, (new Ast(null, $subContext))->withParent($cg->context));
$cg->ts->inject($mutation);
}
}), consume(rtoken('/^(T_\\w+·\\w+|·\\w+|···\\w+)$/'))->onCommit(function (Ast $result) use($cg) {
$expansion = $cg->context->{(string) $result->token()};
if ($expansion instanceof Token) {
$cg->ts->inject(TokenStream::fromSequence($expansion));
} elseif (is_array($expansion) && \count($expansion)) {
$tokens = [];
array_walk_recursive($expansion, function (Token $token) use(&$tokens) {
$tokens[] = $token;
});
$cg->ts->inject(TokenStream::fromSlice($tokens));
}
}), any()))->parse($cg->ts);
$cg->ts->reset();
if ($this->cloaked) {
traverse(either(consume(token(Token::CLOAKED))->onCommit(function (Ast $result) use($cg) {
$cg->ts->inject(TokenStream::fromSourceWithoutOpenTag((string) $result->token()));
}), any()))->parse($cg->ts);
$cg->ts->reset();
}
return $cg->ts;
}
/**
* get substream
* @param int $from
* @param int $to
*/
public function get($from, $to)
{
$tokenStream = new TokenStream();
$tokenStream->append(array_slice($this->tokens, $from, $to - $from + 1));
return $tokenStream;
}
function testPush()
{
$ts = TokenStream::fromSource("<?php 1 2 3 ");
$ts->push(new Token(T_LNUMBER, '4'));
$ts->push(new Token(T_WHITESPACE, ' '));
$ts->push(new Token(T_LNUMBER, '5'));
$ts->push(new Token(T_WHITESPACE, ' '));
$this->assertEquals('<?php 1 2 3 4 5 ', (string) $ts);
$ts = TokenStream::fromSource('<?php ');
$ts->push(new Token(T_STRING, 'A'));
$ts->push(new Token(T_WHITESPACE, ' '));
$ts->push(new Token(T_STRING, 'B'));
$ts->push(new Token(T_WHITESPACE, ' '));
$this->assertEquals('<?php A B ', (string) $ts);
}
