/** * Test the `setSource` and `getSource` accessors. */ public function testSourceAccessors() { $source = 'a string'; $stream = new TokenStream(); $stream->setSource($source); $this->assertSame($source, $stream->getSource()); }
/** * Transform the input string into a token stream. * * @param string $input The string input to tokenize. * @return TokenStream The resulting token stream. */ private function tokenize($input) { $stream = new TokenStream(); $stream->setSource($input); $pattern = '/' . implode('|', $this->lexemes) . '/'; $flags = PREG_SPLIT_NO_EMPTY | PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_OFFSET_CAPTURE; $matches = preg_split($pattern, $input, -1, $flags); foreach ($matches as $match) { $value = strtolower($match[0]); if ($value[0] == "'" || $value[0] == '"' || $value == 'true' || $value == 'false' || $value == 'null' || is_numeric($value)) { $code = self::T_VALUE; } else { if (isset($this->constMap[$value])) { $code = $this->constMap[$value]; } else { if (preg_match('/[a-z0-9_]+/', $value)) { $code = self::T_NAME; } else { if (ctype_space($value)) { continue; } else { $code = self::T_NONE; } } } } $stream->push(new AnnotationToken($code, $match[0], $match[1])); } return $stream; }
/** * Transform the input string into a token stream. * * @param string $input The string input to tokenize. * @return TokenStream The resulting token stream. */ private function tokenize($input) { $stream = new TokenStream(); $stream->setSource($input); $pattern = '/' . implode('|', $this->lexemes) . '/'; $flags = PREG_SPLIT_NO_EMPTY | PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_OFFSET_CAPTURE; $matches = preg_split($pattern, $input, -1, $flags); foreach ($matches as $match) { $value = strtolower($match[0]); if (is_numeric($value)) { $code = self::T_NUMBER; } else { if (isset($this->constTokenMap[$value])) { $code = $this->constTokenMap[$value]; } else { if (ctype_space($value)) { continue; } else { $code = self::T_NONE; } } } $stream->push(new Token($code, $match[0], $match[1])); } return $stream; }