/**
* Used to split a string of text in the language given by $locale into
* space separated words. Ex: "acontinuousstringofwords" becomes
* "a continuous string of words". It operates by scanning from the end of
* the string to the front and splitting on the longest segment that is a
* word.
*
* @param string $segment string to make into a string of space separated
* words
* @param string $locale IANA tag used to look up dictionary filter to
* use to do this segmenting
* @return string space separated words
*/
static function reverseMaximalMatch($segment, $locale)
{
$len = mb_strlen($segment);
$cur_pos = $len - 1;
if ($cur_pos < 1) {
return $segment;
}
$out_segment = "";
$word_len = 2;
$char_added = mb_substr($segment, $cur_pos, 1);
$word_guess = $char_added;
$is_ascii = mb_check_encoding($char_added, 'ASCII');
$was_space = trim($char_added) == "";
$added = false;
$suffix_backup = -1;
$suffix_check = true;
while ($cur_pos >= 0) {
$old_word_guess = $word_guess;
$cur_pos--;
$char_added = mb_substr($segment, $cur_pos, 1);
$is_space = trim($char_added) == "";
if ($is_space && $was_space) {
continue;
}
if (!$is_space) {
$word_guess = $char_added . $word_guess;
$was_space = false;
} else {
$was_space = true;
}
if ($suffix_check) {
$is_suffix = NWordGrams::ngramsContains("*" . $word_guess, $locale, "segment");
} else {
$is_suffix = false;
}
$suffix_check = true;
$is_ascii = $is_ascii && mb_check_encoding($char_added, 'ASCII') && !$is_space;
if ($is_suffix || $is_ascii) {
$word_len++;
$added = false;
if ($is_suffix) {
$suffix_backup = $cur_pos;
$suffix_guess = $word_guess;
}
} else {
if ($suffix_backup >= 0) {
$cur_pos = $suffix_backup;
$suffix_backup = -1;
$suffix_check = false;
$word_guess = $suffix_guess;
$word_len = strlen($word_guess);
} else {
if (NWordGrams::ngramsContains($word_guess, $locale, "segment") || $is_space) {
$out_segment .= " " . strrev($word_guess);
$cur_pos--;
$suffix_backup = -1;
$word_len = 1;
$word_guess = mb_substr($segment, $cur_pos, $word_len);
$is_ascii = mb_check_encoding($word_guess, 'ASCII');
$was_space = trim($char_added) == "";
$word_len++;
$added = true;
} else {
$word_len = 1;
$suffix_backup = -1;
$out_segment .= " " . strrev($old_word_guess);
$word_guess = mb_substr($segment, $cur_pos, $word_len);
$is_ascii = mb_check_encoding($word_guess, 'ASCII');
$was_space = trim($char_added) == "";
$word_len++;
$added = true;
}
}
}
}
if (!$added) {
if (NWordGrams::ngramsContains($old_word_guess, $locale, "segment") || mb_check_encoding($old_word_guess, 'ASCII')) {
$out_segment .= " " . strrev($old_word_guess);
} else {
$front = mb_substr($old_word_guess, 0, 1);
$back = mb_substr($old_word_guess, 1);
$out_segment .= " " . strrev($front) . " " . strrev($back);
}
}
$out_segment = strrev($out_segment);
return $out_segment;
}
