<?php
include_once "Tokenizer.php";
$tokenizer = new Tokenizer();
$tokenizer->addDefinition('+', '[+]');
$tokenizer->addDefinition('-', '[-]');
$tokenizer->addDefinition('*', '[*]');
$tokenizer->addDefinition('/', '[/]');
$tokenizer->addDefinition('=', '[=]');
$tokenizer->addDefinition('(', '[(]');
$tokenizer->addDefinition(')', '[)]');
$tokenizer->addDefinition('command', 'set');
$tokenizer->addDefinition('number', '[0-9]+([.][0-9]+)?');
$tokenizer->addDefinition('identifier', '[a-zA-Z_]\\w*');
$tokenizer->ignore(' ');
// "set" will be a "command", because it was added first and takes priority over "identifier"
// "sets" will be an "identifier", because longer matches have priority, 4 characters vs 3 characters
$tokens = $tokenizer->tokenize("set x = (14 + y) - 12.5 + sets");
var_dump($tokens);
