function testEscapeToken()
{
$tokens = array(array('NAME', '<>&', false), array('NAME', 'no html entities here', false), array('NAME', '<>&', false), array('NAME', '<>&', true), array('NAME', 'no html entities here', true), array('NAME', '<>&', true));
foreach ($tokens as $t) {
$escaped = Utils::escapeToken($t);
// name should be unchanged
assert($t[0] === $escaped[0]);
if ($t[2]) {
$expected = $t[1];
// already escaped, should be unchanged
} else {
$expected = Utils::escapeString($t[1]);
}
assert($escaped[1] === $expected);
assert($escaped[2]);
}
}
public function main()
{
// we're aiming to handle context, unified and normal diff all at once here
// because it doesn't really seem that hard.
$child = null;
$lastIndex = -1;
while (!$this->eos()) {
$index = $this->pos();
assert($index > $lastIndex);
$lastIndex = $index;
assert($this->bol());
$tok = null;
if ($this->scan('/diff\\s.*$/m') !== null) {
$tok = 'KEYWORD';
} elseif ($this->scan($this->patterns['range']) !== null) {
// normal, context and unified ranges
$tok = 'DIFF_RANGE';
} elseif ($this->scan("/-{3}[ \t]*\$/m")) {
$tok = null;
} elseif ($this->scan('/(?:\\**|=*|\\w.*)$/m') !== null) {
$tok = 'KEYWORD';
} elseif ($this->scan("@[+\\-\\*]{3}(\\s+([^\\s]*)([ \t]|\$))?.*@m") !== null) {
// this is a header line which may contain a file path. If it does,
// update the child scanner according to its extension.
$m = $this->matchGroups();
// unified uses +++, context uses *
if ($m[0][0] === '+' || $m[0][0] === '*') {
$tok = 'DIFF_HEADER_NEW';
} else {
$tok = 'DIFF_HEADER_OLD';
}
if (isset($m[2])) {
$filename = preg_replace('@.*\\\\/@', '', $m[2]);
$child = self::getChildScanner($filename);
}
} elseif ($this->scan('/\\\\.*/') !== null) {
$tok = null;
} elseif ($this->scan($this->patterns['codeblock']) !== null) {
// this is actual source code.
// we're going to format this here.
// we're going to extract the block, and try to re-assemble it as
// verbatim code, then highlight it via a child scanner, then split up
// the lines, re-apply the necessary prefixes (e.g. + or -) to them,
// and store them as being a DIFF_ token.
// we have to do it like this, rather than line by line, otherwise
// multiline tokens aren't going to work properly. There's stilla risk
// that the diff will be fragmented such the child scanner gets it
// wrong but that can't be helped.
// TODO restructure this so the complicated bits aren't done if there's
// no child scanner to pass it down to
$block = $this->match();
if (!strlen($block)) {
assert(0);
}
$lines = explode("\n", $block);
$verbatim = array();
$verbatim_ = '';
$types = array();
$prefixes = array();
foreach ($lines as $l) {
if (!strlen($l) || $l[0] === ' ') {
$types[] = 'DIFF_UNCHANGED';
} elseif ($l[0] === '+' || $l[0] === '>') {
$types[] = 'DIFF_NEW';
} elseif ($l[0] === '!' || $l[0] === '<' || $l[0] === '-') {
$types[] = 'DIFF_OLD';
} else {
assert(0);
}
$prefixes[] = isset($l[0]) ? $l[0] : '';
$verbatim_[] = substr($l, 1);
}
$verbatim = implode("\n", $verbatim_);
$escaped = false;
$tagged;
if ($child !== null) {
$c = new $child();
$c->init();
$c->string($verbatim);
$c->main();
$tagged = $c->tagged();
$escaped = true;
} else {
$tagged = $verbatim;
}
$exp = explode("\n", $tagged);
assert(count($exp) === count($prefixes));
foreach ($exp as $i => $v) {
$t = $types[$i];
// if the sub-scanner escaped the line, we also need to escape the
// prefix for consistency
$prefix = $prefixes[$i];
if ($escaped) {
$prefix = Utils::escapeString($prefix);
}
$text = $prefix . $v;
$this->record($text, $t, $escaped);
if ($i < count($exp) - 1) {
$this->record("\n", null);
}
}
if ($this->eol()) {
$this->record($this->get(), null);
}
continue;
} else {
$this->scan('/.*/');
}
// previous else clause can capture empty strings
if ($this->match() !== '') {
$this->record($this->match(), $tok);
}
assert($this->eol());
// consume newline
if (!$this->eos()) {
$this->record($this->get(), null);
}
}
}
/**
* @brief Returns the XML representation of the token stream
*
* This function triggers the generation of the XML output.
* @return An XML-string which represents the tokens recorded by the scanner.
*/
public function tagged()
{
$out = '';
// call stream filters.
foreach ($this->streamFilters as $f) {
$this->tokens = call_user_func($f[1], $this->tokens);
}
foreach ($this->tokens as $t) {
$type = $t[0];
// speed is roughly 10% faster if we process the filters inside this
// loop instead of separately.
if (isset($this->filters[$type])) {
foreach ($this->filters[$type] as $filter) {
$t = call_user_func($filter[1], $t);
}
}
list($type, $string, $esc) = $t;
if (!$esc) {
$string = Utils::escapeString($string);
}
if ($type !== null) {
$out .= Utils::tagBlock($type, $string);
} else {
$out .= $string;
}
}
return $out;
}
/**
* Recursive function to collapse the token tree into XML
* @internal
*/
protected function collapseTokenTree($node)
{
$text = '';
foreach ($node['children'] as $c) {
if (is_string($c)) {
$text .= Utils::escapeString($c);
} else {
$text .= $this->collapseTokenTree($c);
}
}
$tokenName = $node['token_name'];
$token = array($node['token_name'], $text, true);
$token_ = $this->ruleMapperFilter(array($token));
$token = $token_[0];
if (isset($this->filters[$tokenName])) {
foreach ($this->filters[$tokenName] as $filter) {
$token = call_user_func($filter[1], $token);
}
}
list($tokenName, $text, ) = $token;
return $tokenName === null ? $text : Utils::tagBlock($tokenName, $text);
}