/**
* Return a list of tokens from a given template_string.
*
* @return array
*/
public function tokenize()
{
$in_tag = False;
$result = array();
foreach (preg_split(DjaBase::getReTag(), $this->template_string, -1, PREG_SPLIT_DELIM_CAPTURE) as $bit) {
if ($bit) {
$result[] = $this->createToken($bit, $in_tag);
}
$in_tag = !$in_tag;
}
return $result;
}
/**
* Return a list of tokens from a given template_string
*
* @return array
*/
public function tokenize()
{
$result = array();
$upto = 0;
$matches = new PyReFinditer(DjaBase::getReTag(), $this->template_string);
/** @var $match pyReMatchObject */
foreach ($matches as $match) {
list($start, $end) = $match->span();
if ($start > $upto) {
$result[] = $this->createToken(py_slice($this->template_string, $upto, $start), False, array($upto, $start));
$upto = $start;
}
$result[] = $this->createToken(py_slice($this->template_string, $start, $end), True, array($start, $end));
$upto = $end;
}
$last_bit = py_slice($this->template_string, $upto);
if ($last_bit) {
$result[] = $this->createToken($last_bit, False, array($upto, $upto + strlen($last_bit)));
}
return $result;
}
