/**
*/
protected function buildTokenizerMethod(Syntax $syntax)
{
$while = new WhileStatement(new RawBlock('strlen($data) !== 0'));
$while->setBody($whileBody = new Container());
if ($syntax->isIgnoreWhitespace()) {
$whileBody->addLine(new RawBlock('$data = trim($data);'));
}
$whileBody->addLine(new RawBlock('$token = $this->getNextToken($data);'));
$whileBody->addLine(new RawBlock('$data = substr($data, strlen($token[1]));'));
$whileBody->addLine(new RawBlock('$tokens[] = $token;'));
$methodBody = new Container();
$methodBody->addLine(new RawBlock('$tokens = [];'));
$methodBody->addLine($while);
$methodBody->addLine(new RawBlock('$tokens[] = array(' . $syntax->getAlias('__END__') . ', \'\');'));
$methodBody->addLine('return $tokens;');
$method = new MethodGenerator('tokenizer', ['data'], MethodGenerator::FLAG_PROTECTED);
$method->setBody($methodBody);
$this->class->addMethodFromGenerator($method);
}
