/**
* Lexical scanner.
* @param string
* @return void
*/
private function tokenize($s)
{
if (!self::$regexp) {
self::$regexp = '~' . implode('|', self::$patterns) . '~mA';
}
$s = str_replace("\r", '', $s);
$s = strtr($s, "\t", ' ');
$s = "\n" . $s . "\n";
// first is required by "Indent", last is required by parse-error check
$this->input = $s;
$this->tokens = String::split($s, self::$regexp, PREG_SPLIT_NO_EMPTY);
if (end($this->tokens) !== "\n") {
// unable to parse
$this->n = key($this->tokens);
$this->error();
}
}
private function tokenize($s)
{
if (!self::$regexp) {
self::$regexp = '~' . implode('|', self::$patterns) . '~mA';
}
$s = str_replace("\r", '', $s);
$s = strtr($s, "\t", ' ');
$s = "\n" . $s . "\n";
$this->input = $s;
$this->tokens = preg_split(self::$regexp, $s, -1, PREG_SPLIT_NO_EMPTY | PREG_SPLIT_DELIM_CAPTURE);
if (end($this->tokens) !== "\n") {
$this->n = key($this->tokens);
$this->error();
}
}
