private function scanCharPositions($text) { $charPositions = array(); $skippedCharPositions = array(); for ($i = 0; $i < strlen($text); $i++) { if (!StringUtil::isWhitespace($text[$i])) { if (empty($charPositions)) { $charPositions[] = $i; } elseif (in_array($i - 1, $charPositions) || in_array($i - 1, $skippedCharPositions)) { $skippedCharPositions[] = $i; continue; } else { $charPositions[] = $i; } } } return $charPositions; }
public function testIsWhitespace() { $this->assertEquals(true, StringUtil::isWhitespace(' ')); $this->assertEquals(true, StringUtil::isWhitespace("\t")); $this->assertEquals(true, StringUtil::isWhitespace("\n")); }
public static function findToken($text, $position) { if ($position < strlen($text) - 1 && !StringUtil::isWhitespace(substr($text, $position + 1, 1))) { $nextWs = StringUtil::getNextWhitespace($text, $position); $prevWs = StringUtil::getPrevWhitespace($text, $position); $tokenStart = $prevWs === false ? 0 : $prevWs + 1; $tokenEnd = $nextWs === false ? strlen($text) : $nextWs; $token = substr($text, $tokenStart, $tokenEnd - $tokenStart); // strip trailing . if (!empty($token) && in_array($token[strlen($token) - 1], self::$standardEosChars)) { $token = substr($token, 0, strlen($token) - 1); $span = new Span($tokenStart, $tokenEnd - 1); } else { $span = new Span($tokenStart, $tokenEnd); } return $span; } }