/**
* @param string $input
* @param callable $callback
* @throws \AerialShip\Lex\Error\UnknownTokenException when unable to match a token from input string
*/
public function tokenizeAsync($input, callable $callback)
{
$offset = 0;
$count = 0;
$matches = null;
while (strlen($input)) {
$anyMatch = false;
foreach ($this->config->getAllTokens() as $token) {
if (preg_match($token->getRegex(), $input, $matches)) {
$str = $matches[0];
$len = strlen($str);
if (strlen($token->getToken()) > 0) {
call_user_func($callback, new Token($token->getToken(), $str, $offset, $count));
$count++;
}
$input = substr($input, $len);
$anyMatch = true;
$offset += $len;
break;
}
}
if (!$anyMatch) {
throw new UnknownTokenException($offset);
}
}
call_user_func($callback, new EOF($offset, $count));
}
/**
* @test
*/
public function shouldReturnArrayOfTokenDefnWhenGetAllTokensCalled()
{
$config = new LexConfig(array('a' => 'aaa', 'b' => 'bbb'));
$arrTokens = $config->getAllTokens();
$this->assertInternalType('array', $arrTokens);
$this->assertContainsOnlyInstancesOf('AerialShip\\Lex\\Config\\TokenDefn', $arrTokens);
}
/**
* @test
*/
public function shouldWork()
{
$config = new LexConfig(new YamlFileConfig(__DIR__ . '/config1.yml'));
$arrTokens = $config->getAllTokens();
$this->assertCount(6, $arrTokens);
$this->assertEquals("|^\\s|", $arrTokens[0]->getRegex());
$this->assertEquals('', $arrTokens[0]->getToken());
$this->assertEquals("|^\\d+|", $arrTokens[1]->getRegex());
$this->assertEquals('number', $arrTokens[1]->getToken());
$this->assertEquals("|^\\+|", $arrTokens[2]->getRegex());
$this->assertEquals('plus', $arrTokens[2]->getToken());
$this->assertEquals("|^-|", $arrTokens[3]->getRegex());
$this->assertEquals('minus', $arrTokens[3]->getToken());
$this->assertEquals("|^\\*|", $arrTokens[4]->getRegex());
$this->assertEquals('mul', $arrTokens[4]->getToken());
$this->assertEquals("|^/|", $arrTokens[5]->getRegex());
$this->assertEquals('div', $arrTokens[5]->getToken());
}