/**
* Loop through the stream, pulling maximum type length each time, find the largest type that matches and create a
* token, then move on length characters
*
* @return Iterator
*/
public function getTokens()
{
fseek($this->stream, 0);
$this->buffer = new StreamBuffer($this->stream, static::BUFFER_SIZE, $this->minLength);
$this->buffer->read();
/** @var Token $last */
$last = null;
while (!$this->buffer->isEof()) {
foreach ($this->state->match($this->buffer) as $token) {
if ($token[0] == Token::T_BOM) {
$this->changeEncoding($token[1]);
}
$this->state = $this->state->getNextState($token[0]);
// merge tokens together to condense T_CONTENT tokens
if ($token[0] == Token::T_CONTENT) {
if (!is_null($last)) {
$last[1] .= $token[1];
$last[3] = strlen($last[1]);
} else {
$last = $token;
}
} else {
if (!is_null($last)) {
(yield $last);
$last = null;
}
(yield $token);
}
$this->buffer->move($token[3]);
$this->buffer->read();
}
}
if (!is_null($last)) {
(yield $last);
}
fclose($this->stream);
}
