private function _process(TokenIterator $tokens)
 {
     $context = new Context($this);
     $result = new Result($tokens->getSource(), $tokens->current());
     for ($tokens->next(); $tokens->valid(); $tokens->next()) {
         if (!$tokens->current()->process($context, $this, $result, $tokens)) {
             break;
         }
     }
     return $result;
 }
 protected function processEnd(Context $context, Language $language, Result $result, TokenIterator $tokens)
 {
     $this->setStart($result->getStart());
     if ($this->_start->postProcess) {
         $source = substr($tokens->getSource(), $this->_start->pos - $tokens->getOffset(), $this->_start->getLength());
         $tokens = $this->_start->inject->tokenize($source, $result->getTokens(), $this->_start->pos, Language::EMBEDDED_BY_PARENT);
         $result->exchangeArray($this->_start->inject->parse($tokens)->getTokens());
     }
     # closing unclosed tokens
     foreach (array_reverse($context->stack, true) as $id => $name) {
         $end = new Token($name, ['pos' => $this->pos]);
         $tokens[$id]->setEnd($end);
         $result->append($end);
     }
     $result->append($this);
     return false;
 }
 public function testTokenReturning()
 {
     $tokens = [new Token(null, ['token.name', 'pos' => 15]), new Token(null, ['token.name', 'pos' => 25])];
     $iterator = new TokenIterator($tokens, 'source');
     $this->assertEquals($tokens, $iterator->getTokens());
 }