Example #1
0
 /**
  * Add word to word-array
  * This function should be used to make sure CJK sequences are split up in the right way
  *
  * @param 	array		Array of accumulated words
  * @param 	string		Complete Input string from where to extract word
  * @param 	integer		Start position of word in input string
  * @param 	integer		The Length of the word string from start position
  * @return 	void
  * @todo Define visibility
  */
 public function addWords(&$words, &$wordString, $start, $len)
 {
     // Get word out of string:
     $theWord = substr($wordString, $start, $len);
     // Get next chars unicode number and find type:
     $bc = 0;
     $cp = $this->utf8_ord($theWord, $bc);
     list($cType) = $this->charType($cp);
     // If string is a CJK sequence we follow this algorithm:
     /*
     		DESCRIPTION OF (CJK) ALGORITHMContinuous letters and numbers make up words. Spaces and symbols
     		separate letters and numbers into words. This is sufficient for
     		all western text.CJK doesn't use spaces or separators to separate words, so the only
     		way to really find out what constitutes a word would be to have a
     		dictionary and advanced heuristics. Instead, we form pairs from
     		consecutive characters, in such a way that searches will find only
     		characters that appear more-or-less the right sequence. For example:ABCDE => AB BC CD DEThis works okay since both the index and the search query is split
     		in the same manner, and since the set of characters is huge so the
     		extra matches are not significant.(Hint taken from ZOPEs chinese user group)[Kasper: As far as I can see this will only work well with or-searches!]
     */
     if ($cType == 'cjk') {
         // Find total string length:
         $strlen = $this->csObj->utf8_strlen($theWord);
         // Traverse string length and add words as pairs of two chars:
         for ($a = 0; $a < $strlen; $a++) {
             if ($strlen == 1 || $a < $strlen - 1) {
                 $words[] = $this->csObj->utf8_substr($theWord, $a, 2);
             }
         }
     } else {
         // Normal "single-byte" chars:
         // Remove chars:
         foreach ($this->lexerConf['removeChars'] as $skipJoin) {
             $theWord = str_replace($this->csObj->UnumberToChar($skipJoin), '', $theWord);
         }
         // Add word:
         $words[] = $theWord;
     }
 }
 /**
  * @test
  */
 public function utf8_strlenForNonEmptyAsciiOnlyStringReturnsNumberOfCharacters()
 {
     $this->assertEquals(10, $this->subject->utf8_strlen('good omens'));
 }