示例#1
0
 public function testTokenSortingEndPrecedesStart()
 {
     $tokens = [];
     $tokens[0] = $token = $this->_factory->create('token.1', ['pos' => 2, 'length' => 0]);
     $tokens[1] = $token->getEnd();
     $tokens[2] = $token = $this->_factory->create('token.3', ['pos' => 5, 'length' => 4]);
     $tokens[3] = $token->getEnd();
     $tokens[4] = $token = $this->_factory->create('token.2', ['pos' => 2, 'length' => 3]);
     $tokens[5] = $token->getEnd();
     $list = new UnprocessedTokens();
     $list->add($tokens[0]);
     $list->add($tokens[2]);
     $list->add($tokens[4]);
     $list->sort();
     $this->assertEquals([$tokens[0], $tokens[1], $tokens[4], $tokens[5], $tokens[2], $tokens[3]], array_values($list->toArray()));
 }
示例#2
0
 /**
  * Tokenize source
  *
  * @param                    $source
  *
  * @param int                $offset
  * @param array|\Traversable $additional
  *
  * @param bool               $embedded
  *
  * @return UnprocessedTokens
  */
 private function _tokens($source, $offset = 0, $additional = [], $embedded = false)
 {
     $result = new UnprocessedTokens();
     /** @var Language $language */
     foreach ($this->_rules($embedded) as $rule) {
         foreach ($rule->match($source) as $token) {
             $result->add($token, $offset);
         }
     }
     return $result->batch($additional);
 }