function _test_escape_token($token) { $escaped = LuminousUtils::escape_token($token); // name should be unchanged assert($token[0] === $escaped[0]); $expected = $token[2] ? $token[1] : LuminousUtils::escape_string($token[1]); assert($escaped[1] === $expected); assert($escaped[2]); }
function main() { // we're aiming to handle context, unified and normal diff all at once here // because it doesn't really seem that hard. $child = null; $last_index = -1; while (!$this->eos()) { $index = $this->pos(); assert($index > $last_index); $last_index = $index; assert($this->bol()); $tok = null; if ($this->scan('/diff\\s.*$/m') !== null) { $tok = 'KEYWORD'; } elseif ($this->scan($this->patterns['range']) !== null) { $tok = 'DIFF_RANGE'; } elseif ($this->scan("/-{3}[ \t]*\$/m")) { $tok = null; } elseif ($this->scan('/(?:\\**|=*|\\w.*)$/m') !== null) { $tok = 'KEYWORD'; } elseif ($this->scan("@[+\\-\\*]{3}(\\s+([^\\s]*)([ \t]|\$))?.*@m") !== null) { $m = $this->match_groups(); // unified uses +++, context uses * if ($m[0][0] === '+' || $m[0][0] === '*') { $tok = 'DIFF_HEADER_NEW'; } else { $tok = 'DIFF_HEADER_OLD'; } if (isset($m[2])) { $filename = preg_replace('@.*\\\\/@', '', $m[2]); $child = self::get_child_scanner($filename); } } elseif ($this->scan('/\\\\.*/') !== null) { $tok = null; } elseif ($this->scan($this->patterns['codeblock']) !== null) { // this is actual source code. // we're going to format this here. // we're going to extract the block, and try to re-assemble it as // verbatim code, then highlight it via a child scanner, then split up // the lines, re-apply the necessary prefixes (e.g. + or -) to them, // and store them as being a DIFF_ token. // we have to do it like this, rather than line by line, otherwise // multiline tokens aren't going to work properly. There's stilla risk // that the diff will be fragmented such the child scanner gets it // wrong but that can't be helped. // TODO restructure this so the complicated bits aren't done if there's // no child scanner to pass it down to $block = $this->match(); if (!strlen($block)) { assert(0); } $lines = explode("\n", $block); $verbatim = array(); $verbatim_ = ''; $types = array(); $prefixes = array(); foreach ($lines as $l) { if (!strlen($l) || $l[0] === ' ') { $types[] = 'DIFF_UNCHANGED'; } elseif ($l[0] === '+' || $l[0] === '>') { $types[] = 'DIFF_NEW'; } elseif ($l[0] === '!' || $l[0] === '<' || $l[0] === '-') { $types[] = 'DIFF_OLD'; } else { assert(0); } $prefixes[] = isset($l[0]) ? $l[0] : ''; $verbatim_[] = substr($l, 1); } $verbatim = implode("\n", $verbatim_); $escaped = false; $tagged; if ($child !== null) { $c = new $child(); $c->init(); $c->string($verbatim); $c->main(); $tagged = $c->tagged(); $escaped = true; } else { $tagged = $verbatim; } $exp = explode("\n", $tagged); assert(count($exp) === count($prefixes)); foreach ($exp as $i => $v) { $t = $types[$i]; // if the sub-scanner escaped the line, we also need to escape the // prefix for consistency $prefix = $prefixes[$i]; if ($escaped) { $prefix = LuminousUtils::escape_string($prefix); } $text = $prefix . $v; $this->record($text, $t, $escaped); if ($i < count($exp) - 1) { $this->record("\n", null); } } if ($this->eol()) { $this->record($this->get(), null); } continue; } else { $this->scan('/.*/'); } // previous else clause can capture empty strings if ($this->match() !== '') { $this->record($this->match(), $tok); } assert($this->eol()); // consume newline if (!$this->eos()) { $this->record($this->get(), null); } } }
/** * Recursive function to collapse the token tree into XML * @internal */ protected function collapse_token_tree($node) { $text = ''; foreach ($node['children'] as $c) { if (is_string($c)) { $text .= LuminousUtils::escape_string($c); } else { $text .= $this->collapse_token_tree($c); } } $token_name = $node['token_name']; $token = array($node['token_name'], $text, true); $token_ = $this->rule_mapper_filter(array($token)); $token = $token_[0]; if (isset($this->filters[$token_name])) { foreach ($this->filters[$token_name] as $filter) { $token = call_user_func($filter[1], $token); } } list($token_name, $text, ) = $token; return $token_name === null ? $text : LuminousUtils::tag_block($token_name, $text); }