/** * Return a list of tokens from a given template_string. * * @return array */ public function tokenize() { $in_tag = False; $result = array(); foreach (preg_split(DjaBase::getReTag(), $this->template_string, -1, PREG_SPLIT_DELIM_CAPTURE) as $bit) { if ($bit) { $result[] = $this->createToken($bit, $in_tag); } $in_tag = !$in_tag; } return $result; }
/** * Return a list of tokens from a given template_string * * @return array */ public function tokenize() { $result = array(); $upto = 0; $matches = new PyReFinditer(DjaBase::getReTag(), $this->template_string); /** @var $match pyReMatchObject */ foreach ($matches as $match) { list($start, $end) = $match->span(); if ($start > $upto) { $result[] = $this->createToken(py_slice($this->template_string, $upto, $start), False, array($upto, $start)); $upto = $start; } $result[] = $this->createToken(py_slice($this->template_string, $start, $end), True, array($start, $end)); $upto = $end; } $last_bit = py_slice($this->template_string, $upto); if ($last_bit) { $result[] = $this->createToken($last_bit, False, array($upto, $upto + strlen($last_bit))); } return $result; }