/**
* This file is part of the sfSearch package.
* (c) Carl Vondrick <*****@*****.**>
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
*/
require dirname(__FILE__) . '/../../bootstrap/unit.php';
require 'lexer/xfLexemeBuilder.class.php';
require 'lexer/xfLexeme.class.php';
require 'util/xfException.class.php';
$t = new lime_test(20, new lime_output_color());
$b = new xfLexemeBuilder('symfony pròject');
$t->is($b->getPosition(), -1, '->getPosition() is negative when processing has not started.');
$t->is($b->getLexemes(), array(), '->getLexemes() is empty when processing has not started.');
$t->is($b->getCharacter(), null, '->getCharacter() is null when processing has not started.');
$b->next();
$t->is($b->getPosition(), 0, '->next() advances the pointer');
$t->is($b->getCharacter(), 's', '->getCharacter() returns the current pointer.');
for ($x = 0; $x < 4; $x++) {
$b->addToLexeme($b->getCharacter());
$b->next();
}
$b->setType('foobar');
$b->commit();
$t->is($b->count(), 1, '->addToLexeme(), ->commit() create lexemes');
$t->is($b->getLexeme(0)->getLexeme(), 'symf', '->getLexeme() returns the lexeme by index.');
$t->is($b->getLexeme(0)->getType(), 'foobar', '->commit() sets the appropriate type');
$t->is($b->getLexeme(0)->getPosition(), 4, '->commit() sets the appropriate position');
for ($x = 0; $x < 3; $x++) {
/**
* Tokenizes the input.
*
* @param string $string
*/
public function tokenize($string, $encoding = 'utf-8')
{
$string = trim($string, "\r\n\t ");
if ($string === '') {
return array();
}
$this->builder = new xfLexemeBuilder($string, $encoding, 'xfLexemeLucene');
$this->setupFiniteStateMachine();
$this->process();
return $this->builder->getLexemes();
}
