function _test_escape_token($token)
{
$escaped = LuminousUtils::escape_token($token);
// name should be unchanged
assert($token[0] === $escaped[0]);
$expected = $token[2] ? $token[1] : LuminousUtils::escape_string($token[1]);
assert($escaped[1] === $expected);
assert($escaped[2]);
}
function main()
{
// we're aiming to handle context, unified and normal diff all at once here
// because it doesn't really seem that hard.
$child = null;
$last_index = -1;
while (!$this->eos()) {
$index = $this->pos();
assert($index > $last_index);
$last_index = $index;
assert($this->bol());
$tok = null;
if ($this->scan('/diff\\s.*$/m') !== null) {
$tok = 'KEYWORD';
} elseif ($this->scan($this->patterns['range']) !== null) {
$tok = 'DIFF_RANGE';
} elseif ($this->scan("/-{3}[ \t]*\$/m")) {
$tok = null;
} elseif ($this->scan('/(?:\\**|=*|\\w.*)$/m') !== null) {
$tok = 'KEYWORD';
} elseif ($this->scan("@[+\\-\\*]{3}(\\s+([^\\s]*)([ \t]|\$))?.*@m") !== null) {
$m = $this->match_groups();
// unified uses +++, context uses *
if ($m[0][0] === '+' || $m[0][0] === '*') {
$tok = 'DIFF_HEADER_NEW';
} else {
$tok = 'DIFF_HEADER_OLD';
}
if (isset($m[2])) {
$filename = preg_replace('@.*\\\\/@', '', $m[2]);
$child = self::get_child_scanner($filename);
}
} elseif ($this->scan('/\\\\.*/') !== null) {
$tok = null;
} elseif ($this->scan($this->patterns['codeblock']) !== null) {
// this is actual source code.
// we're going to format this here.
// we're going to extract the block, and try to re-assemble it as
// verbatim code, then highlight it via a child scanner, then split up
// the lines, re-apply the necessary prefixes (e.g. + or -) to them,
// and store them as being a DIFF_ token.
// we have to do it like this, rather than line by line, otherwise
// multiline tokens aren't going to work properly. There's stilla risk
// that the diff will be fragmented such the child scanner gets it
// wrong but that can't be helped.
// TODO restructure this so the complicated bits aren't done if there's
// no child scanner to pass it down to
$block = $this->match();
if (!strlen($block)) {
assert(0);
}
$lines = explode("\n", $block);
$verbatim = array();
$verbatim_ = '';
$types = array();
$prefixes = array();
foreach ($lines as $l) {
if (!strlen($l) || $l[0] === ' ') {
$types[] = 'DIFF_UNCHANGED';
} elseif ($l[0] === '+' || $l[0] === '>') {
$types[] = 'DIFF_NEW';
} elseif ($l[0] === '!' || $l[0] === '<' || $l[0] === '-') {
$types[] = 'DIFF_OLD';
} else {
assert(0);
}
$prefixes[] = isset($l[0]) ? $l[0] : '';
$verbatim_[] = substr($l, 1);
}
$verbatim = implode("\n", $verbatim_);
$escaped = false;
$tagged;
if ($child !== null) {
$c = new $child();
$c->init();
$c->string($verbatim);
$c->main();
$tagged = $c->tagged();
$escaped = true;
} else {
$tagged = $verbatim;
}
$exp = explode("\n", $tagged);
assert(count($exp) === count($prefixes));
foreach ($exp as $i => $v) {
$t = $types[$i];
// if the sub-scanner escaped the line, we also need to escape the
// prefix for consistency
$prefix = $prefixes[$i];
if ($escaped) {
$prefix = LuminousUtils::escape_string($prefix);
}
$text = $prefix . $v;
$this->record($text, $t, $escaped);
if ($i < count($exp) - 1) {
$this->record("\n", null);
}
}
if ($this->eol()) {
$this->record($this->get(), null);
}
continue;
} else {
$this->scan('/.*/');
}
// previous else clause can capture empty strings
if ($this->match() !== '') {
$this->record($this->match(), $tok);
}
assert($this->eol());
// consume newline
if (!$this->eos()) {
$this->record($this->get(), null);
}
}
}
/**
* Recursive function to collapse the token tree into XML
* @internal
*/
protected function collapse_token_tree($node)
{
$text = '';
foreach ($node['children'] as $c) {
if (is_string($c)) {
$text .= LuminousUtils::escape_string($c);
} else {
$text .= $this->collapse_token_tree($c);
}
}
$token_name = $node['token_name'];
$token = array($node['token_name'], $text, true);
$token_ = $this->rule_mapper_filter(array($token));
$token = $token_[0];
if (isset($this->filters[$token_name])) {
foreach ($this->filters[$token_name] as $filter) {
$token = call_user_func($filter[1], $token);
}
}
list($token_name, $text, ) = $token;
return $token_name === null ? $text : LuminousUtils::tag_block($token_name, $text);
}
