public function testCallGetNextStateWithAnInvalidTokenWillThrowAnException() { $tokenStore = m::mock(TokenStoreInterface::class); $tokenStore->shouldReceive('getTokens')->andReturn([]); $state = new State($tokenStore); $state->addStateTarget(Token::T_CONTENT, $state); static::assertSame($state, $state->getNextState(Token::T_CONTENT)); static::expectException(RuntimeException::class); $state->getNextState(Token::T_ESCAPE); }
/** * @param TokenStoreInterface $tokenStore * * @return State The default starting state */ public function buildStates(TokenStoreInterface $tokenStore) { $initial = new State($tokenStore, State::S_INITIAL_TOKENS); $any = new State($tokenStore, State::S_ANY_TOKENS); $inQuote = new State($tokenStore, State::S_IN_QUOTE_TOKENS); $inEscape = new State($tokenStore, State::S_IN_ESCAPE_TOKENS); $inQuoteEscape = new State($tokenStore, State::S_IN_QUOTE_ESCAPE_TOKENS); $initial->addStateTarget(Token::T_ANY & ~Token::T_QUOTE & ~Token::T_ESCAPE, $any); $initial->addStateTarget(Token::T_QUOTE, $inQuote); $initial->addStateTarget(Token::T_ESCAPE, $inEscape); // generate state mapping $any->addStateTarget(Token::T_ANY & ~Token::T_QUOTE & ~Token::T_ESCAPE, $any); $any->addStateTarget(Token::T_QUOTE, $inQuote); $any->addStateTarget(Token::T_ESCAPE, $inEscape); $inQuote->addStateTarget(Token::T_CONTENT | Token::T_DOUBLE_QUOTE, $inQuote); $inQuote->addStateTarget(Token::T_QUOTE, $any); $inQuote->addStateTarget(Token::T_ESCAPE, $inQuoteEscape); $inEscape->addStateTarget(Token::T_CONTENT, $any); $inQuoteEscape->addStateTarget(Token::T_CONTENT, $inQuote); return $initial; }
/** * Loop through the stream, pulling maximum type length each time, find the largest type that matches and create a * token, then move on length characters * * @return Iterator */ public function getTokens() { fseek($this->stream, 0); $this->buffer = new StreamBuffer($this->stream, static::BUFFER_SIZE, $this->minLength); $this->buffer->read(); /** @var Token $last */ $last = null; while (!$this->buffer->isEof()) { foreach ($this->state->match($this->buffer) as $token) { if ($token[0] == Token::T_BOM) { $this->changeEncoding($token[1]); } $this->state = $this->state->getNextState($token[0]); // merge tokens together to condense T_CONTENT tokens if ($token[0] == Token::T_CONTENT) { if (!is_null($last)) { $last[1] .= $token[1]; $last[3] = strlen($last[1]); } else { $last = $token; } } else { if (!is_null($last)) { (yield $last); $last = null; } (yield $token); } $this->buffer->move($token[3]); $this->buffer->read(); } } if (!is_null($last)) { (yield $last); } fclose($this->stream); }