/** * Stem and then look up the word * @param string $token */ public function stem($token) { if (in_array($token, $this->whiteList)) { return $token; } return $this->spell->suggest($this->stemmer->stem($token))[0]; }
/** * Apply a stemmer * @param IStemmer $stemmer * @return \TextAnalysis\Documents\TokensDocument */ public function applyStemmer(IStemmer $stemmer) { foreach ($this->tokens as &$token) { $token = $stemmer->stem($token); } //filter null tokens and re-index $this->tokens = array_values(array_filter($this->tokens)); return $this; }
/** * Apply a stemmer * @param IStemmer $stemmer * @param boolean $removeNulls * @return \TextAnalysis\Documents\TokensDocument */ public function applyStemmer(IStemmer $stemmer, $removeNulls = true) { foreach ($this->tokens as &$token) { $token = $stemmer->stem($token); } if ($removeNulls) { //filter null tokens and re-index $this->tokens = array_values(array_filter($this->tokens)); } return $this; }
/** * Stem and then look up the word * @param string $token */ public function stem($token) { return $this->spell->suggest($this->stemmer->stem($token))[0]; }