function calculate($line) { global $parser; if (!strlen($line)) return; try { $parser->reset(); foreach(tokenize($line) as $t) { if (is_numeric($t)) $parser->eat('num', doubleval($t)); else if (ctype_alpha($t)) $parser->eat('var', $t); else $parser->eat("'$t'", null); } $parser->eat_eof(); } catch (parse_error $e) { echo $e->getMessage(), "\n"; } }
function parse($data) { // Tokenize data into an array - splits data into multiple elements // by CR, LF, and TAB, and removes all curly braces $tokens = tokenize($data, '/[\\r\\n\\t]/', '/[\\{\\}]/'); // ...then for each element in the array, split the element up into // another array, splitting the data into multiple elements by spaces, // while keeping quoted lines together; then remove ($) and (") from // each element to clean up array_walk($tokens, function (&$element, $key) { $element = tokenize($element, '/([^\\"]\\S*|\\".+?\\")\\s*/', '/[\\$\\"]/'); }); return $tokens; }
<?php function tokenize($str, $token) { $tokenizedStrArr = array(); $tokStr = strtok($str, $token); while ($tokStr !== false) { $tokenizedStrArr[] = $tokStr; $tokStr = strtok($token); } return $tokenizedStrArr; } var_dump(tokenize('foobarbaz', 'foo')); var_dump(tokenize('foobarbaz', 'bar')); var_dump(tokenize('foobarbaz', 'baz')); var_dump(tokenize('foobarbaz', 'foobar')); var_dump(tokenize('foobarbaz', 'barbaz')); var_dump(tokenize('foobarbaz', 'foobaz'));
$def = ""; continue; } if ($collect) { $def .= $tokens[$i]; } } indent("\${$tokens[1]}={$def};\n", $depth); } break; } else { /* HANDLE CURSOR DECLARATIONS */ $select = ""; $curs_name = $tokens[1]; $proc_cursors[$curs_name]['vars'] = array(); $tokens = tokenize($stmt, false); #we need to preserve format for the cursor SQL foreach ($tokens as $token) { if ($token != 'SELECT' && !$select) { continue; } #variables in the cursor get turned into PHP variables #TODO look in the list of args and vars for the token just in case if (substr($token, 0, 2) == "v_") { $proc_cursors[$curs_name]['vars'][] = $token; $token = "\${$token}"; } $select .= $token; } $proc_cursors[$curs_name]['sql'] = $select; //print_r($proc_cursors);
<?php if ($argc > 1) { array_shift($argv); foreach ($argv as $code) { tokenize($code); } } else { $code = fread(STDIN, 1024); tokenize($code); } function tokenize($code) { foreach (token_get_all($code) as $t) { if (is_array($t)) { $token_name = token_name($t[0]); $text = preg_replace_callback('#[^\\x21-\\x7f]#', 'chr_replace', $t[1]); } else { $token_name = ''; $text = preg_replace_callback('#[^\\x21-\\x7f]#', 'chr_replace', $t); } echo sprintf("%-30s %s\n", $token_name, $text); } } function chr_replace($matches) { return '\\x' . sprintf('%02x', ord($matches[0])); }
function interpretline($str, $anstype, $countcnt) { $str .= ';'; $bits = array(); $lines = array(); $len = strlen($str); $cnt = 0; $ifloc = -1; $elseloc = array(); $forloc = -1; $whereloc = -1; $lastsym = ''; $lasttype = -1; $closeparens = 0; $symcnt = 0; //get tokens from tokenizer $syms = tokenize($str, $anstype, $countcnt); $k = 0; $symlen = count($syms); //$lines holds lines of code; $bits holds symbols for the current line. while ($k < $symlen) { list($sym, $type) = $syms[$k]; //first handle stuff that would use last symbol; add it if not needed if ($sym == '^' && $lastsym != '') { //found a ^: convert a^b to safepow(a,b) $bits[] = 'safepow('; $bits[] = $lastsym; $bits[] = ','; $k++; list($sym, $type) = $syms[$k]; $closeparens++; //triggers to close safepow after next token $lastsym = '^'; $lasttype = 0; } else { if ($sym == '!' && $lasttype != 0 && $lastsym != '' && $syms[$k + 1][0] != '=') { //convert a! to factorial(a), avoiding if(!a) and a!=b $bits[] = 'factorial('; $bits[] = $lastsym; $bits[] = ')'; $sym = ''; } else { if ($lasttype == 2 && $type == 4 && substr($lastsym, 0, 5) == 'root(') { $bits[] = substr($lastsym, 0, -1) . ','; $sym = substr($sym, 1); $lasttype = 0; } else { //add last symbol to stack if ($lasttype != 7 && $lasttype != -1) { $bits[] = $lastsym; } } } } if ($closeparens > 0 && $lastsym != '^' && $lasttype != 0) { //close safepow. lasttype!=0 to get a^-2 to include - while ($closeparens > 0) { $bits[] = ')'; $closeparens--; } //$closeparens = false; } if ($sym == '=' && $ifloc == -1 && $whereloc == -1 && $lastsym != '<' && $lastsym != '>' && $lastsym != '!' && $lastsym != '=' && $syms[$k + 1][0] != '=') { //if equality equal (not comparison), and before if/where. //check for commas to the left, convert $a,$b = to list($a,$b) = $j = count($bits) - 1; $hascomma = false; while ($j >= 0) { if ($bits[$j] == ',') { $hascomma = true; break; } $j--; } if ($hascomma) { array_unshift($bits, "list("); array_push($bits, ')'); $hascomma = false; } } else { if ($type == 7) { //end of line if ($lasttype == '7' || $lasttype == -1) { //nothing exciting, so just continue $k++; continue; } //check for for, if, where and rearrange bits if needed if ($forloc > -1) { //convert for($i=a..b) {todo} $j = $forloc; while ($bits[$j][0] != '{' && $j < count($bits)) { $j++; } $cond = implode('', array_slice($bits, $forloc + 1, $j - $forloc - 1)); $todo = implode('', array_slice($bits, $j)); //might be $a..$b or 3.*.4 (remnant of implicit handling) if (preg_match('/^\\s*\\(\\s*(\\$\\w+)\\s*\\=\\s*(-?\\d+|\\$[\\w\\[\\]]+)\\s*\\.\\s?\\.\\s*(-?\\d+|\\$[\\w\\[\\]]+)\\s*\\)\\s*$/', $cond, $matches)) { $forcond = array_slice($matches, 1, 3); $bits = array("if (is_nan({$forcond[2]}) || is_nan({$forcond[1]})) {echo 'part of for loop is not a number';} else {for ({$forcond[0]}=intval({$forcond[1]});{$forcond[0]}<=round(floatval({$forcond[2]}),0);{$forcond[0]}++) " . $todo . "}"); } else { echo _('error with for code.. must be "for ($var=a..b) {todo}" where a and b are whole numbers or variables only'); return 'error'; } } else { if ($ifloc == 0) { //this is if at beginning of line, form: if ($a==3) {todo} $j = 0; while ($bits[$j][0] != '{' && $j < count($bits)) { $j++; } if ($j == count($bits)) { echo _('need curlys for if statement at beginning of line'); return 'error'; } $cond = implode('', array_slice($bits, 1, $j - 1)); if (count($elseloc) == 0) { $todo = implode('', array_slice($bits, $j)); } else { $todo = implode('', array_slice($bits, $j, $elseloc[0][0] - $j)); } $out = "if ({$cond}) {$todo}"; for ($i = 0; $i < count($elseloc); $i++) { $j = $elseloc[$i][0]; while ($bits[$j][0] != '{' && $j < count($bits)) { $j++; } if ($j == count($bits)) { echo _('need curlys for else statement'); return 'error'; } if ($i == count($elseloc) - 1) { $todo = implode('', array_slice($bits, $j)); } else { $todo = implode('', array_slice($bits, $j, $elseloc[$i + 1][0] - $j)); } if ($j - $elseloc[$i][0] == 1) { //no condition if ($elseloc[$i][1] == 'elseif') { echo _('need condition for elseif'); return 'error'; } $out .= " else {$todo}"; } else { //has condition $cond = implode('', array_slice($bits, $elseloc[$i][0] + 1, $j - $elseloc[$i][0] - 1)); $out .= " else if ({$cond}) {$todo}"; } } $bits = array($out); } else { if (count($elseloc) > 0) { echo _('else used without leading if statement'); return 'error'; } } } if ($whereloc > 0) { //handle $a = rand() where ($a==b) if ($ifloc > -1 && $ifloc < $whereloc) { echo _('line of type $a=b if $c==0 where $d==0 is invalid'); return 'error'; } $wheretodo = implode('', array_slice($bits, 0, $whereloc)); if ($ifloc > -1) { //handle $a = rand() where ($a==b) if ($c==0) $wherecond = implode('', array_slice($bits, $whereloc + 1, $ifloc - $whereloc - 1)); $ifcond = implode('', array_slice($bits, $ifloc + 1)); if ($countcnt == 1) { //if outermost $bits = array('if (' . $ifcond . ') {$wherecount[0]=0;$wherecount[' . $countcnt . ']=0;do{' . $wheretodo . ';$wherecount[' . $countcnt . ']++;$wherecount[0]++;} while (!(' . $wherecond . ') && $wherecount[' . $countcnt . ']<200 && $wherecount[0]<1000); if ($wherecount[' . $countcnt . ']==200) {echo "where not met in 200 iterations";}; if ($wherecount[0]>=1000 && $wherecount[0]<2000) {echo "nested where not met in 1000 iterations";}}'); } else { $bits = array('if (' . $ifcond . ') {$wherecount[' . $countcnt . ']=0;do{' . $wheretodo . ';$wherecount[' . $countcnt . ']++;$wherecount[0]++;} while (!(' . $wherecond . ') && $wherecount[' . $countcnt . ']<200 && $wherecount[0]<1000); if ($wherecount[' . $countcnt . ']==200) {echo "where not met in 200 iterations";$wherecount[0]=5000;} }'); } } else { $wherecond = implode('', array_slice($bits, $whereloc + 1)); if ($countcnt == 1) { $bits = array('$wherecount[0]=0;$wherecount[' . $countcnt . ']=0;do{' . $wheretodo . ';$wherecount[' . $countcnt . ']++;$wherecount[0]++;} while (!(' . $wherecond . ') && $wherecount[' . $countcnt . ']<200 && $wherecount[0]<1000); if ($wherecount[' . $countcnt . ']==200) {echo "where not met in 200 iterations";}; if ($wherecount[0]>=1000 && $wherecount[0]<2000 ) {echo "nested where not met in 1000 iterations";}'); } else { $bits = array('$wherecount[' . $countcnt . ']=0;do{' . $wheretodo . ';$wherecount[' . $countcnt . ']++;$wherecount[0]++;} while (!(' . $wherecond . ') && $wherecount[' . $countcnt . ']<200 && $wherecount[0]<1000); if ($wherecount[' . $countcnt . ']==200) {echo "where not met in 200 iterations";$wherecount[0]=5000;}; '); } } } else { if ($ifloc > 0) { //handle $a = b if ($c==0) $todo = implode('', array_slice($bits, 0, $ifloc)); $cond = implode('', array_slice($bits, $ifloc + 1)); $bits = array("if ({$cond}) { {$todo} ; }"); } } $forloc = -1; $ifloc = -1; $whereloc = -1; $elseloc = array(); //collapse bits to a line, add to lines array $lines[] = implode('', $bits); $bits = array(); } else { if ($type == 1) { //is var //implict 3$a and $a $b and (3-4)$a if ($lasttype == 3 || $lasttype == 1 || $lasttype == 4) { $bits[] = '*'; } } else { if ($type == 2) { //is func //implicit $v sqrt(2) and 3 sqrt(3) and (2-3)sqrt(4) and sqrt(2)sqrt(3) if ($lasttype == 3 || $lasttype == 1 || $lasttype == 4 || $lasttype == 2) { $bits[] = '*'; } } else { if ($type == 3) { //is num //implicit 2 pi and $var pi if ($lasttype == 3 || $lasttype == 1) { $bits[] = '*'; } } else { if ($type == 4) { //is parens //implicit 3(4) (5)(3) $v(2) if ($lasttype == 3 || $lasttype == 4 || $lasttype == 1) { $bits[] = '*'; } } else { if ($type == 8) { //is control //mark location of control symbol if ($sym == 'if') { $ifloc = count($bits); } else { if ($sym == 'where') { $whereloc = count($bits); } else { if ($sym == 'for') { $forloc = count($bits); } else { if ($sym == 'else' || $sym == 'elseif') { $elseloc[] = array(count($bits), $sym); } } } } } else { if ($type == 9) { //is error //tokenizer returned an error token - exit current loop with error return 'error'; } else { if ($sym == '-' && $lastsym == '/') { //paren 1/-2 to 1/(-2) //avoid bug in PHP 4 where 1/-2*5 = -0.1 but 1/(-2)*5 = -2.5 $bits[] = '('; $closeparens++; } } } } } } } } } $lastsym = $sym; $lasttype = $type; $cnt++; $k++; } //if no explicit end-of-line at end of bits if (count($bits) > 0) { $lines[] = implode('', $bits); } //collapse to string return implode(";\n", $lines); }
function bb2rss($bb) { $text = ""; $omit = false; // Create a mixed array of BB tags and content $tokens = tokenize($bb); // Loop through each token for ($i = 0; $i < count($tokens); $i++) { $token = $tokens[$i]; // Remove content after equal signs if (strpos($token, "=")) { $token = strstr($token, "=", true); } // Process the token switch ($token) { case "[b]": case "[/b]": case "[i]": case "[/i]": case "[u]": case "[/u]": case "[s]": case "[/s]": case "[color": case "[/color]": case "[size": case "[/size]": case "[center]": case "[/center]": break; case "[list]": case "[ul]": case "[ol]": $text .= "(list)"; $omit = true; case "[*]": case "[li]": case "[/li]": break; case "[/list]": case "[/ul]": case "[/ol]": $omit = false; break; case "[table]": $text .= "(table)"; $omit = true; case "[tr]": case "[th]": case "[/th]": case "[td]": case "[/td]": case "[/tr]": break; case "[/table]": $omit = false; break; case "[quote]": case "[quote": $text .= "(quote)"; $omit = true; case "[/quote]": $omit = false; break; case "[code]": $text .= "(code)"; $omit = true; break; case "[/code]": $omit = false; case "[url]": case "[url": case "[/url]": break; case "[img]": case "[img": case "[img width": $text .= "(image)"; $omit = true; break; case "[/img]": $omit = false; break; case "[youtube]": $text .= "(viedo)"; $omit = true; break; case "[/youtube]": $omit = false; break; default: if (!$omit) { $text .= $tokens[$i]; } } } // Replace new lines with spaces $text = str_replace("\n", " ", $text); return $text; }
<?php require "header.php"; require "fungsi.php"; if (isset($_GET['id'])) { $id = $_GET['id']; ?> <body> <div><h2 align='center'>Proses Tokenisasi</h2></div> <div class="dataanggota"> <?php tokenize($id); ?> </div> <?php } else { echo ""; }
fatal_error("Unknown option: {$arg}"); } } if ($name == "") { fatal_error("--name missing"); } if ($input_file == "") { fatal_error("--input-file missing"); } if ($output_dir == "") { fatal_error("--output-dir missing"); } if (($data = file_get_contents($input_file)) === FALSE) { fatal_error("Failed to read input file"); } if (!tokenize($data, $tokens)) { fatal_error("Failed to tokenize"); } $parser = new parse_engine(new ProtoParser()); try { foreach ($tokens as $token) { $parser->eat($token[0], $token[1]); } $parser->eat_eof(); } catch (parse_error $e) { fatal_error("{$input_file}: Parse error: " . $e->getMessage()); } $data = generate_header($name, $parser->semantic["directives"], $parser->semantic["messages"]); if (file_put_contents("{$output_dir}/{$name}.h", $data) === NULL) { fatal_error("{$input_file}: Failed to write .h file"); }
function compile($script, $noheader = false) { global $inlineFlagStack; global $localVarRegistry; global $currentFunction; global $globalVarRegistry; global $pramsRegistry; if (!is_array($inlineFlagStack)) { $inlineFlagStack = array(); } if (!is_array($pramsRegistry)) { $pramsRegistry = array(); } array_push($pramsRegistry, array()); if (!is_array($localVarRegistry)) { $localVarRegistry = array(); } if (!is_array($currentFunction)) { $currentFunction = array(); } if (!is_array($globalVarRegistry)) { $globalVarRegistry = array(); } array_push($currentFunction, "global"); array_push($localVarRegistry, array("", array())); array_push($inlineFlagStack, false); $source = ""; $script = explode("\n", $script); if (!$noheader) { $source .= "/* Generated By jsCc */\n\n"; } if ($script[0] == "/* Generated By jsCc */") { return "[jsCc: error: Line 1] Cannot compile already compiled code."; } $doCompile = true; foreach ($script as $scriptLine) { $scriptLine = stripComments($scriptLine); if ($scriptLine != "") { if ($scriptLine == "[begin js]") { $doCompile = false; continue; } if ($doCompile == true) { $source .= (!$noheader ? "\n// " . $scriptLine . "\n" : "") . compileLine(tokenize($scriptLine)); } else { if ($scriptLine == "[end js]") { $doCompile = true; } else { $source .= $scriptLine . "\n"; } } } } $source = "trainStations = [];\ntrainLines = [];\n" . $source; $source .= "executeTrain();\n"; array_pop($inlineFlagStack); $lvr = array_pop($localVarRegistry); $lvk = array_keys($lvr); foreach ($lvk as $lkey) { $lvars = $lvr[$lkey]; $vars = ""; if (is_array($lvr[$lkey])) { foreach ($lvars as $lvar) { $vars .= "var " . $lvar . " = \"" . $lvar . "\";\n"; } $source = str_replace("@localvars_" . $lkey, $vars, $source); } } array_pop($pramsRegistry); $source = str_replace("\r", "", str_replace("\n", "\\n", str_replace("\"", "\\\"", $source))); return $source; }
function process_stat_dates() { global $start_collection_date; global $end_collection_date; global $given_dates; global $basic_where; $tmp_begin = 0; $tmp_end = 0; $tokens = tokenize($given_dates); foreach ($tokens as $token) { # Find Range (e.g., 2007-07-01 - 2007-08-01) if (preg_match("/(\\d*)(-|\\/)(\\d*)(-|\\/)(\\d*)/", $token, $m)) { if ($tmp_begin == 0) { $tmp_begin = $m[0]; } else { $tmp_end = $m[0]; } } } if ($tmp_begin != 0) { $start_collection_date = "DATE '{$tmp_begin}'"; $end_collection_date = "DATE '{$tmp_end}'"; } }
function read_str($str) { $tokens = tokenize($str); if (count($tokens) === 0) { throw new BlankException(); } return read_form(new Reader($tokens)); }
// with Wiki words later. Wiki words in URL's break things. // URLs preceeded by a '!' are not linked $tmpline = tokenize($tmpline, "!?\\b({$AllowedProtocols}):[^\\s<>\\[\\]\"'()]*[^\\s<>\\[\\]\"'(),.?]", $replacements, $ntokens); while ($oldn < $ntokens) { if ($replacements[$oldn][0] == '!') { $replacements[$oldn] = substr($replacements[$oldn], 1); } else { $replacements[$oldn] = LinkURL($replacements[$oldn]); } $oldn++; } ////////////////////////////////////////////////////////// // Link Wiki words // Wikiwords preceeded by a '!' are not linked $oldn = $ntokens; $tmpline = tokenize($tmpline, "!?{$WikiNameRegexp}", $replacements, $ntokens); while ($oldn < $ntokens) { $old = $replacements[$oldn]; if ($old[0] == '!') { $replacements[$oldn] = substr($old, 1); } elseif (IsWikiPage($dbi, $old)) { $replacements[$oldn] = LinkExistingWikiWord($old); } else { $replacements[$oldn] = LinkUnknownWikiWord($old); } $oldn++; } ////////////////////////////////////////////////////////// // escape HTML metachars $tmpline = str_replace('&', '&', $tmpline); $tmpline = str_replace('>', '>', $tmpline);
// */ //POUR INTERDIRE LE HTML, SUPPRIMER LES // A LA LIGNE CI-DESSUS (2/2) $oldn = $ntokens; $tmpline = tokenize($tmpline, '\[\[', $replacements, $ntokens); while ($oldn < $ntokens) $replacements[$oldn++] = '['; $oldn = $ntokens; $tmpline = tokenize($tmpline, '\[\s*\d+\s*\]', $replacements, $ntokens); while ($oldn < $ntokens) { $num = (int) mb_substr($replacements[$oldn], 1); if (! empty($embedded[$num])) $replacements[$oldn] = $embedded[$num]; $oldn++; } $oldn = $ntokens; $tmpline = tokenize($tmpline, '\[.+?\]', $replacements, $ntokens); while ($oldn < $ntokens) { $link = ParseAndLink($replacements[$oldn]); $replacements[$oldn] = $link['link']; $oldn++; } /* $tmpline = tokenize($tmpline, "!?\b($AllowedProtocols):[^\s<>\[\]\"'()]*[^\s<>\[\]\"'(),.?]", $replacements, $ntokens); while ($oldn < $ntokens) { if($replacements[$oldn][0] == '!') $replacements[$oldn] = mb_substr($replacements[$oldn], 1); else $replacements[$oldn] = LinkURL($replacements[$oldn]); $oldn++; } */
/** * @test * @dataProvider provideInvalidEdn * @expectedException Phlexy\LexingException */ function parseShouldRejectInvalidSyntax($edn) { $data = tokenize($edn); $this->fail(sprintf('Expected lexer to fail on %s, but got: %s', json_encode($edn), print_r($data, true))); }
/** * parse the criteria string according to a subset of the Google search syntax: * * example: this too AND +this OR (these AND "the other") -notthis * * @return array of criterias, each criteria being an array: * array (0 => operator, ' AND ',' OR ',' AND NOT ' * 1 => value, the criteria's string value * 2 => wildcard flag, TRUE if wildcard matching, FALSE for strict matching * 3 => nesting) parentheses nesting level 0..n * * for convenience, the operator of the first criteria is blank. */ function parsecriteria($criteria) { $results = array(); $tokens = array(); $thisresult = array('', '', TRUE, 0); $nesting = 0; //var_dump($criteria); //@@@ //replace html quoting put there by some browsers, then tokenize $tokens = tokenize(str_replace('"', '"', $criteria)); //var_dump($tokens); //@@@ foreach ($tokens as $token) { switch ($token[0]) { case TOKEN_AND: $thisresult[0] = ' AND '; $thisresult[2] = TRUE; //reset wildcard in case of bad syntax break; case TOKEN_OR: $thisresult[0] = ' OR '; $thisresult[2] = TRUE; //reset wildcard in case of bad syntax break; case TOKEN_PLUS: $thisresult[2] = FALSE; break; case TOKEN_NOT: case TOKEN_MINUS: $thisresult[0] .= ' NOT '; //NOT or AND NOT break; case TOKEN_LEFTPAREN: $nesting += 1; $thisresult[2] = TRUE; //reset just in case of bad syntax break; case TOKEN_RIGHTPAREN: $nesting -= 1; $thisresult[2] = TRUE; //reset just in case of bad syntax break; default: // anything else -> output "as is" $thisresult[1] .= $token[1]; $thisresult[3] = $nesting; $results[] = $thisresult; $thisresult = array(' AND ', '', TRUE, 0); break; } } //foreach $tokens return $results; }