function testSimpleTokenizer() { $tokenizer = new Tokenizer\SimpleTokenizer(); $tokens = $tokenizer->tokenize("Test string to tokenize. Why!?, To make sure it works! ... "); $this->assertContains('test', $tokens); $this->assertContains('string', $tokens); $this->assertContains('to', $tokens); $this->assertContains('tokenize', $tokens); $this->assertContains('why', $tokens); $this->assertContains('works', $tokens); }
public function setUp() { $this->index->clear(); $this->store->clear(); $this->assertEquals(0, $this->index->size()); $this->assertEquals(0, $this->store->size()); $tokenizer = new Tokenizer\SimpleTokenizer(); $this->documents = [new Document('A', 'a s d'), new Document('B', 'b n m'), new Document('C', 'c v f')]; foreach ($this->documents as $id => $document) { $document->id = $id; $document->tokens = $tokenizer->tokenize($document->content); $this->store->addDocument($document); } $this->assertEquals(0, $this->index->size()); $this->index = $this->store->buildIndex($this->index); }