/**
* Normalize Token or remove it (if null is returned)
*
* @param Zend_Search_Lucene_Analysis_Token $srcToken
* @return Zend_Search_Lucene_Analysis_Token
*/
public function normalize(Zend_Search_Lucene_Analysis_Token $po_srctoken)
{
$vo_lang_analyzer = new LanguageDetection();
$vs_original_string = $po_srctoken->getTermText();
$vs_lang_code = $vo_lang_analyzer->analyze($vs_original_string);
/* stem text with respect to language that has been detected */
$vo_stemmer = new SnoballStemmer();
if ($vs_lang_code) {
$vs_stemmed_string = $vo_stemmer->stem($vs_original_string, $vs_lang_code);
} else {
/* if language could not be detected, don't do any stemming at all */
$vs_stemmed_string = $vs_original_string;
}
/* build new token to return */
$vo_new_token = new Zend_Search_Lucene_Analysis_Token($vs_stemmed_string, $po_srctoken->getStartOffset(), $po_srctoken->getEndOffset());
$vo_new_token->setPositionIncrement($po_srctoken->getPositionIncrement());
return $vo_new_token;
}
