private function findToken($text, $position)
{
if ($position < strlen($text) - 1) {
$nextWs = StringUtil::getNextWhitespace($text, $position);
$prevWs = StringUtil::getPrevWhitespace($text, $position);
$tokenStart = $prevWs === false ? 0 : $prevWs + 1;
$tokenEnd = $nextWs === false ? strlen($text) : $nextWs;
$token = substr($text, $tokenStart, $tokenEnd - $tokenStart);
$span = new Span($tokenStart, $tokenEnd);
return $span;
}
}
public function find($text)
{
$charPositions = $this->scanCharPositions($text);
$tokens = array();
foreach ($charPositions as $pos) {
$token = StringUtil::findToken($text, $pos);
if ($token !== null) {
$tokens[] = $token;
}
}
foreach ($tokens as $j => $token) {
if (!$this->validator->isValid($token->getCoveredText($text))) {
unset($tokens[$j]);
}
}
return array_values($tokens);
}
private function scanCharPositions($text)
{
$charPositions = array();
$skippedCharPositions = array();
for ($i = 0; $i < strlen($text); $i++) {
if (!StringUtil::isWhitespace($text[$i])) {
if (empty($charPositions)) {
$charPositions[] = $i;
} elseif (in_array($i - 1, $charPositions) || in_array($i - 1, $skippedCharPositions)) {
$skippedCharPositions[] = $i;
continue;
} else {
$charPositions[] = $i;
}
}
}
return $charPositions;
}
public function find($text)
{
$atCharPositions = array();
$tokens = array();
for ($i = 0; $i < strlen($text); $i++) {
if ($text[$i] === '@') {
$atCharPositions[] = $i;
}
}
foreach ($atCharPositions as $pos) {
$token = StringUtil::findToken($text, $pos);
if ($token !== null) {
$tokens[] = $token;
}
}
foreach ($tokens as $j => $token) {
if (!$this->validator->isValid($token->getCoveredText($text))) {
unset($tokens[$j]);
}
}
return array_values($tokens);
}
public function testGetStandardEosChars()
{
$this->assertNotEmpty(StringUtil::getStandardEosChars());
}
public static function findToken($text, $position)
{
if ($position < strlen($text) - 1 && !StringUtil::isWhitespace(substr($text, $position + 1, 1))) {
$nextWs = StringUtil::getNextWhitespace($text, $position);
$prevWs = StringUtil::getPrevWhitespace($text, $position);
$tokenStart = $prevWs === false ? 0 : $prevWs + 1;
$tokenEnd = $nextWs === false ? strlen($text) : $nextWs;
$token = substr($text, $tokenStart, $tokenEnd - $tokenStart);
// strip trailing .
if (!empty($token) && in_array($token[strlen($token) - 1], self::$standardEosChars)) {
$token = substr($token, 0, strlen($token) - 1);
$span = new Span($tokenStart, $tokenEnd - 1);
} else {
$span = new Span($tokenStart, $tokenEnd);
}
return $span;
}
}
