public function testTokenSortingEndPrecedesStart()
{
$tokens = [];
$tokens[0] = $token = $this->_factory->create('token.1', ['pos' => 2, 'length' => 0]);
$tokens[1] = $token->getEnd();
$tokens[2] = $token = $this->_factory->create('token.3', ['pos' => 5, 'length' => 4]);
$tokens[3] = $token->getEnd();
$tokens[4] = $token = $this->_factory->create('token.2', ['pos' => 2, 'length' => 3]);
$tokens[5] = $token->getEnd();
$list = new UnprocessedTokens();
$list->add($tokens[0]);
$list->add($tokens[2]);
$list->add($tokens[4]);
$list->sort();
$this->assertEquals([$tokens[0], $tokens[1], $tokens[4], $tokens[5], $tokens[2], $tokens[3]], array_values($list->toArray()));
}
/**
* Tokenize source
*
* @param $source
*
* @param int $offset
* @param array|\Traversable $additional
*
* @param bool $embedded
*
* @return UnprocessedTokens
*/
private function _tokens($source, $offset = 0, $additional = [], $embedded = false)
{
$result = new UnprocessedTokens();
/** @var Language $language */
foreach ($this->_rules($embedded) as $rule) {
foreach ($rule->match($source) as $token) {
$result->add($token, $offset);
}
}
return $result->batch($additional);
}