PHP Yep\Tokenizer Tokenizer Examples

Example #1

File: TokenizerTest.php Project: YepFoundation/tokenizer

 /**
  * @expectedException \Yep\Tokenizer\UnexpectedTokenException
  */
 public function testTokenizerWithUnexpectedTokenException()
 {
     $tokenizer = new Tokenizer(['TOKEN_UPPER' => '[A-Z]+']);
     $tokenizer->tokenize('FooBAR');
 }

Example #2

File: TokenIteratorTest.php Project: YepFoundation/tokenizer

 protected function prepareTestTokens()
 {
     $tokenizer = new Tokenizer(['TOKEN_A' => 'A', 'TOKEN_B' => 'B', 'TOKEN_C' => 'C', 'TOKEN_D' => 'D', 'TOKEN_E' => 'E']);
     $tokens = $tokenizer->tokenize('ABCDE');
     return $tokens;
 }