/**
* Stem and then look up the word
* @param string $token
*/
public function stem($token)
{
if (in_array($token, $this->whiteList)) {
return $token;
}
return $this->spell->suggest($this->stemmer->stem($token))[0];
}
/**
* Apply a stemmer
* @param IStemmer $stemmer
* @return \TextAnalysis\Documents\TokensDocument
*/
public function applyStemmer(IStemmer $stemmer)
{
foreach ($this->tokens as &$token) {
$token = $stemmer->stem($token);
}
//filter null tokens and re-index
$this->tokens = array_values(array_filter($this->tokens));
return $this;
}
/**
* Apply a stemmer
* @param IStemmer $stemmer
* @param boolean $removeNulls
* @return \TextAnalysis\Documents\TokensDocument
*/
public function applyStemmer(IStemmer $stemmer, $removeNulls = true)
{
foreach ($this->tokens as &$token) {
$token = $stemmer->stem($token);
}
if ($removeNulls) {
//filter null tokens and re-index
$this->tokens = array_values(array_filter($this->tokens));
}
return $this;
}
/**
* Stem and then look up the word
* @param string $token
*/
public function stem($token)
{
return $this->spell->suggest($this->stemmer->stem($token))[0];
}
