public function testSentenceTokenizer() { $tokenizer = new SentenceTokenizer(); $this->assertCount(2, $tokenizer->tokenize("This has some words. Why only 4 words?")); $this->assertCount(2, $tokenizer->tokenize("My name is Yooper. I like programming!")); $this->assertCount(2, $tokenizer->tokenize("My name is Yooper!? I like programming!! !!")); }
/** * * @param type $text */ public function __construct($text) { $tokenizer = new SentenceTokenizer(); $this->sentences = $tokenizer->tokenize($this->normalize($text)); }