function testSimpleTokenizer()
{
$tokenizer = new Tokenizer\SimpleTokenizer();
$tokens = $tokenizer->tokenize("Test string to tokenize. Why!?, To make sure it works! ... ");
$this->assertContains('test', $tokens);
$this->assertContains('string', $tokens);
$this->assertContains('to', $tokens);
$this->assertContains('tokenize', $tokens);
$this->assertContains('why', $tokens);
$this->assertContains('works', $tokens);
}
public function setUp()
{
$this->index->clear();
$this->store->clear();
$this->assertEquals(0, $this->index->size());
$this->assertEquals(0, $this->store->size());
$tokenizer = new Tokenizer\SimpleTokenizer();
$this->documents = [new Document('A', 'a s d'), new Document('B', 'b n m'), new Document('C', 'c v f')];
foreach ($this->documents as $id => $document) {
$document->id = $id;
$document->tokens = $tokenizer->tokenize($document->content);
$this->store->addDocument($document);
}
$this->assertEquals(0, $this->index->size());
$this->index = $this->store->buildIndex($this->index);
}
