/**
* Test the `setSource` and `getSource` accessors.
*/
public function testSourceAccessors()
{
$source = 'a string';
$stream = new TokenStream();
$stream->setSource($source);
$this->assertSame($source, $stream->getSource());
}
/**
* Transform the input string into a token stream.
*
* @param string $input The string input to tokenize.
* @return TokenStream The resulting token stream.
*/
private function tokenize($input)
{
$stream = new TokenStream();
$stream->setSource($input);
$pattern = '/' . implode('|', $this->lexemes) . '/';
$flags = PREG_SPLIT_NO_EMPTY | PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_OFFSET_CAPTURE;
$matches = preg_split($pattern, $input, -1, $flags);
foreach ($matches as $match) {
$value = strtolower($match[0]);
if ($value[0] == "'" || $value[0] == '"' || $value == 'true' || $value == 'false' || $value == 'null' || is_numeric($value)) {
$code = self::T_VALUE;
} else {
if (isset($this->constMap[$value])) {
$code = $this->constMap[$value];
} else {
if (preg_match('/[a-z0-9_]+/', $value)) {
$code = self::T_NAME;
} else {
if (ctype_space($value)) {
continue;
} else {
$code = self::T_NONE;
}
}
}
}
$stream->push(new AnnotationToken($code, $match[0], $match[1]));
}
return $stream;
}
/**
* Transform the input string into a token stream.
*
* @param string $input The string input to tokenize.
* @return TokenStream The resulting token stream.
*/
private function tokenize($input)
{
$stream = new TokenStream();
$stream->setSource($input);
$pattern = '/' . implode('|', $this->lexemes) . '/';
$flags = PREG_SPLIT_NO_EMPTY | PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_OFFSET_CAPTURE;
$matches = preg_split($pattern, $input, -1, $flags);
foreach ($matches as $match) {
$value = strtolower($match[0]);
if (is_numeric($value)) {
$code = self::T_NUMBER;
} else {
if (isset($this->constTokenMap[$value])) {
$code = $this->constTokenMap[$value];
} else {
if (ctype_space($value)) {
continue;
} else {
$code = self::T_NONE;
}
}
}
$stream->push(new Token($code, $match[0], $match[1]));
}
return $stream;
}
