/** * Loop through the stream, pulling maximum type length each time, find the largest type that matches and create a * token, then move on length characters * * @return Iterator */ public function getTokens() { fseek($this->stream, 0); $this->buffer = new StreamBuffer($this->stream, static::BUFFER_SIZE, $this->minLength); $this->buffer->read(); /** @var Token $last */ $last = null; while (!$this->buffer->isEof()) { foreach ($this->state->match($this->buffer) as $token) { if ($token[0] == Token::T_BOM) { $this->changeEncoding($token[1]); } $this->state = $this->state->getNextState($token[0]); // merge tokens together to condense T_CONTENT tokens if ($token[0] == Token::T_CONTENT) { if (!is_null($last)) { $last[1] .= $token[1]; $last[3] = strlen($last[1]); } else { $last = $token; } } else { if (!is_null($last)) { (yield $last); $last = null; } (yield $token); } $this->buffer->move($token[3]); $this->buffer->read(); } } if (!is_null($last)) { (yield $last); } fclose($this->stream); }