public function testTokenSortingEndPrecedesStart() { $tokens = []; $tokens[0] = $token = $this->_factory->create('token.1', ['pos' => 2, 'length' => 0]); $tokens[1] = $token->getEnd(); $tokens[2] = $token = $this->_factory->create('token.3', ['pos' => 5, 'length' => 4]); $tokens[3] = $token->getEnd(); $tokens[4] = $token = $this->_factory->create('token.2', ['pos' => 2, 'length' => 3]); $tokens[5] = $token->getEnd(); $list = new UnprocessedTokens(); $list->add($tokens[0]); $list->add($tokens[2]); $list->add($tokens[4]); $list->sort(); $this->assertEquals([$tokens[0], $tokens[1], $tokens[4], $tokens[5], $tokens[2], $tokens[3]], array_values($list->toArray())); }
/** * Tokenize source * * @param $source * * @param int $offset * @param array|\Traversable $additional * * @param bool $embedded * * @return UnprocessedTokens */ private function _tokens($source, $offset = 0, $additional = [], $embedded = false) { $result = new UnprocessedTokens(); /** @var Language $language */ foreach ($this->_rules($embedded) as $rule) { foreach ($rule->match($source) as $token) { $result->add($token, $offset); } } return $result->batch($additional); }