function bootstrapParser() { global $argv; // Not using autoloader. Not needed for such a small piece of code. // why require_once is here and not on top of the file? because I would only need it here. Lets keep the memory // footprint small and include only the bare minimum code we have to. require_once 'LogParser.php'; $parser = new LogParser(__DIR__ . '/../data/sample.log'); $parser->parse($argv[1], $argv[2]); return true; }
/** * Regex Tester * * @param string $type type * @param string $regex regex * @param array $match matchers * @param array $types typers * @param array $logs logs * @param boolean $headers display header * @param string $multiline multiline field * * @return string html */ function test($type, $regex, $match, $types, $logs, $headers = true, $multiline = '') { $r = '<h4>' . $type . '</h4>'; $r .= '<pre>'; $r .= $headers === true ? '<strong>Regex</strong>: ' . $regex . "\n" : ''; $r .= $headers === true ? '<strong>Log </strong>: ' . $logs . "\n" : ''; $r .= $headers === true ? "\n" : ''; $logs = array_reverse(explode("\n", $logs)); $rank = 0; $size = count(strval(count($logs))) + 2; $blan = str_pad('', $size); $buffer = array(); foreach ($logs as $log) { $tokens = LogParser::parseLine($regex, $match, $log, $types); if (is_array($tokens)) { $rank++; $disp = $headers ? '' : str_pad('#' . $rank, $size); $maxlength = 0; foreach ($tokens as $token => $value) { $maxlength = max($maxlength, strlen($token)); } $r .= $headers ? '' : '<strong>' . $disp . $log . "</strong>\n"; foreach ($tokens as $token => $value) { if (substr($token, 0, 3) === 'pml') { continue; } $r .= $blan . '<strong>' . str_pad($token, $maxlength) . '</strong>: ' . $value; if ($token === $multiline) { if (count($buffer) > 0) { $buffer = array_reverse($buffer); foreach ($buffer as $append) { $r .= "\n" . $blan . str_pad('', $maxlength) . ' ' . $append; } } } $r .= "\n"; } $r .= "\n"; $buffer = array(); } else { $buffer[] = $log; } } $r .= '</pre>'; return $r; }
<?php include 'config.php'; include 'classes' . DS . 'Encounter.php'; include 'classes' . DS . 'Bosses.php'; include $include_path . 'classes' . DS . 'logparser.class.php'; $filepath = $include_path . 'combat_logs' . DS . $argv[1] . ".gz"; //Connect to mysql. mysql_connect($mysql_host, $mysql_user, $mysql_pass); mysql_select_db($mysql_db); //Create the parser. $parser = new LogParser($argv[1], 1, -1); //Combat factors. $v = new stdClass(); $v->combat = false; $v->day = 1; $v->date = 0; $v->timestamp = 0; $v->ts = null; $v->ts_last = -1; $v->combat_actions = array(2, 3, 4, 10, 15, 16, 19, 22, 23, 26); //Log and line stuff. $v->line = 0; $v->byte_count = 0; $v->killed_bosses = array(); $params = array(); $encounter = null; //Now we need to parse it. while ($parser->parseLine(true)) { $v->line++; $v->byte_count += $parser->getBytes();
public function testParseWithValidFormatAndData() { $tmpHandle = tmpfile(); $fileContent = 'at=info method=GET path=/api/users/0 dyno=web.13 connect=10ms service=20ms '; // even if I have lines with invalid format as long as these don't match the requestUri and method // I wouldn't need to worry $fileContent .= PHP_EOL . 'at=info method=GET path=/api/users/1/get_friends_score dyno=web.1'; for ($i = 1; $i <= 20; $i++) { $fileContent .= PHP_EOL . 'at=info method=GET path=/api/users/1 dyno=web.' . $i % 3; $fileContent .= ' connect=' . rand(1, 9) * $i . 'ms service=' . rand(20, 50) * $i . 'ms '; } fwrite($tmpHandle, $fileContent); $metaData = stream_get_meta_data($tmpHandle); $tmpFilename = $metaData['uri']; $parser = new LogParser($tmpFilename); ob_start(); $parser->parse('GET', '/api/users/{user_id}'); fclose($tmpHandle); $contents = ob_get_clean(); $this->assertContains('Number of times request was made: 21', $contents); $this->assertContains('Most active Dyno(s): web.1, web.2', $contents); $this->assertContains('Least active Dyno(s): web.13', $contents); $this->assertContains('Min. Response Time: ', $contents); $this->assertContains('Max. Response Time: ', $contents); $this->assertContains('Mean Response Time: ', $contents); $this->assertContains('Median Response Time: ', $contents); $this->assertContains('Mode Response Time(s): ', $contents); }
$count = isset($_GET['count']) ? $_GET['count'] : (isset($files[$file_id]['max']) ? $files[$file_id]['max'] : LOGS_MAX); $timeout = isset($_GET['timeout']) ? $_GET['timeout'] : MAX_SEARCH_LOG_TIME; $regex = $files[$file_id]['format']['regex']; $match = $files[$file_id]['format']['match']; $types = $files[$file_id]['format']['types']; $multiline = isset($files[$file_id]['format']['multiline']) ? $files[$file_id]['format']['multiline'] : ''; $exclude = isset($files[$file_id]['format']['exclude']) ? $files[$file_id]['format']['exclude'] : array(); $title = isset($files[$file_id]['format']['export_title']) ? $files[$file_id]['format']['export_title'] : ''; $file_path = $files[$file_id]['path']; $start_offset = 0; $start_from = SEEK_END; $load_more = false; $old_lastline = ''; $data_to_parse = filesize($file_path); $full = true; $logs = LogParser::getNewLines($regex, $match, $types, $tz, $count, $exclude, $file_path, $start_offset, $start_from, $load_more, $old_lastline, $multiline, $search, $data_to_parse, $full, $timeout); /* |-------------------------------------------------------------------------- | Error while getting logs |-------------------------------------------------------------------------- | */ if (!is_array($logs)) { http500(); } /* |-------------------------------------------------------------------------- | Return |-------------------------------------------------------------------------- | */
function apache_get_config($type, $file, $software, $counter) { $file_json_encoded = json_encode($file); ///////////////////////////////////////////////////////// // Apache error files are not the same on 2.2 and 2.4 // ///////////////////////////////////////////////////////// if ($type == 'error') { // Write a line of log and try to guess the format $remain = 10; $test = 0; error_log('Pimp my Log has been successfully configured with Apache'); foreach (LogParser::getLinesFromBottom($file, 10) as $line) { $test = @preg_match('|^\\[(.*) (.*) (.*) (.*):(.*):(.*)\\.(.*) (.*)\\] \\[(.*):(.*)\\] \\[pid (.*)\\] .*\\[client (.*):(.*)\\] (.*)(, referer: (.*))*$|U', $line); if ($test === 1) { break; } $remain--; if ($remain <= 0) { break; } } ///////////////////// // Error 2.4 style // ///////////////////// if ($test === 1) { return <<<EOF \t\t"{$software}{$counter}": { \t\t\t"display" : "Apache Error #{$counter}", \t\t\t"path" : {$file_json_encoded}, \t\t\t"refresh" : 5, \t\t\t"max" : 10, \t\t\t"notify" : true, \t\t\t"format" : { \t\t\t\t"type" : "HTTPD 2.4", \t\t\t\t"regex" : "|^\\\\[(.*) (.*) (.*) (.*):(.*):(.*)\\\\.(.*) (.*)\\\\] \\\\[(.*):(.*)\\\\] \\\\[pid (.*)\\\\] .*\\\\[client (.*):(.*)\\\\] (.*)(, referer: (.*))*\$|U", \t\t\t\t"export_title" : "Log", \t\t\t\t"match" : { \t\t\t\t\t"Date" : { \t\t\t\t\t\t"M" : 2, \t\t\t\t\t\t"d" : 3, \t\t\t\t\t\t"H" : 4, \t\t\t\t\t\t"i" : 5, \t\t\t\t\t\t"s" : 6, \t\t\t\t\t\t"Y" : 8 \t\t\t\t\t}, \t\t\t\t\t"IP" : 12, \t\t\t\t\t"Log" : 14, \t\t\t\t\t"Severity" : 10, \t\t\t\t\t"Referer" : 16 \t\t\t\t}, \t\t\t\t"types": { \t\t\t\t\t"Date" : "date:H:i:s", \t\t\t\t\t"IP" : "ip:http", \t\t\t\t\t"Log" : "preformatted", \t\t\t\t\t"Severity" : "badge:severity", \t\t\t\t\t"Referer" : "link" \t\t\t\t}, \t\t\t\t"exclude": { \t\t\t\t\t"Log": ["\\/PHP Stack trace:\\/", "\\/PHP *[0-9]*\\\\. \\/"] \t\t\t\t} \t\t\t} \t\t} EOF; } else { return <<<EOF \t\t"{$software}{$counter}": { \t\t\t"display" : "Apache Error #{$counter}", \t\t\t"path" : {$file_json_encoded}, \t\t\t"refresh" : 5, \t\t\t"max" : 10, \t\t\t"notify" : true, \t\t\t"format" : { \t\t\t\t"type" : "HTTPD 2.2", \t\t\t\t"regex" : "|^\\\\[(.*)\\\\] \\\\[(.*)\\\\] (\\\\[client (.*)\\\\] )*((?!\\\\[client ).*)(, referer: (.*))*\$|U", \t\t\t\t"export_title" : "Log", \t\t\t\t"match" : { \t\t\t\t\t"Date" : 1, \t\t\t\t\t"IP" : 4, \t\t\t\t\t"Log" : 5, \t\t\t\t\t"Severity" : 2, \t\t\t\t\t"Referer" : 7 \t\t\t\t}, \t\t\t\t"types": { \t\t\t\t\t"Date" : "date:H:i:s", \t\t\t\t\t"IP" : "ip:http", \t\t\t\t\t"Log" : "preformatted", \t\t\t\t\t"Severity" : "badge:severity", \t\t\t\t\t"Referer" : "link" \t\t\t\t}, \t\t\t\t"exclude": { \t\t\t\t\t"Log": ["\\/PHP Stack trace:\\/", "\\/PHP *[0-9]*\\\\. \\/"] \t\t\t\t} \t\t\t} \t\t} EOF; } } else { if ($type == 'access') { return <<<EOF \t\t"{$software}{$counter}": { \t\t\t"display" : "Apache Access #{$counter}", \t\t\t"path" : {$file_json_encoded}, \t\t\t"refresh" : 0, \t\t\t"max" : 10, \t\t\t"notify" : false, \t\t\t"format" : { \t\t\t\t"type" : "NCSA", \t\t\t\t"regex" : "|^((\\\\S*) )*(\\\\S*) (\\\\S*) (\\\\S*) \\\\[(.*)\\\\] \\"(\\\\S*) (.*) (\\\\S*)\\" ([0-9]*) (.*)( \\"(.*)\\" \\"(.*)\\"( [0-9]*/([0-9]*))*)*\$|U", \t\t\t\t"export_title" : "URL", \t\t\t\t"match" : { \t\t\t\t\t"Date" : 6, \t\t\t\t\t"IP" : 3, \t\t\t\t\t"CMD" : 7, \t\t\t\t\t"URL" : 8, \t\t\t\t\t"Code" : 10, \t\t\t\t\t"Size" : 11, \t\t\t\t\t"Referer" : 13, \t\t\t\t\t"UA" : 14, \t\t\t\t\t"User" : 5, \t\t\t\t\t"\\u03bcs" : 16 \t\t\t\t}, \t\t\t\t"types": { \t\t\t\t\t"Date" : "date:H:i:s", \t\t\t\t\t"IP" : "ip:geo", \t\t\t\t\t"URL" : "txt", \t\t\t\t\t"Code" : "badge:http", \t\t\t\t\t"Size" : "numeral:0b", \t\t\t\t\t"Referer" : "link", \t\t\t\t\t"UA" : "ua:{os.name} {os.version} | {browser.name} {browser.version}\\/100", \t\t\t\t\t"\\u03bcs" : "numeral:0,0" \t\t\t\t}, \t\t\t\t"exclude": { \t\t\t\t\t"URL": ["\\/favicon.ico\\/", "\\/\\\\.pml\\\\.php.*\$\\/"], \t\t\t\t\t"CMD": ["\\/OPTIONS\\/"] \t\t\t\t} \t\t\t} \t\t} EOF; } } }
# command-line or server line-break output define('LINE_BREAK', PHP_SAPI === 'cli' ? "\n" : '<br>'); # command-line usage if (PHP_SAPI === 'cli') { if (@(!$_SERVER['argv'][1])) { $sUsage = "\n " . basename($_SERVER['argv'][0], '.php') . "\n\n\tusage: " . basename($_SERVER['argv'][0], '.php') . " <filename>\n\n"; die($sUsage); } $sFile = $_SERVER['argv'][1]; } else { $sFile = 'access.log'; } if (!file_exists($sFile)) { die(LINE_BREAK . $sFile . ' does not exist in this directory!' . LINE_BREAK . LINE_BREAK); } else { $oLP = new LogParser($sFile); echo $oLP->generateReport(); } ################################################################################## class LogParser { /** * Apache log file parser. * * Coded for PHP 5.4+ * Tested on Debian, CentOS, and Windows (XAMPP) Apache log files. * * Example usage: * * php -f logparser.php /var/log/apache2/access.log * php -f logparser.php /var/log/httpd/access_log
echo "File '{$logFile}' is too old, it will be skiped.\n"; } else { if (array_key_exists($logFileTime, $sortedLogFiles)) { echo "File '{$logFile}' has the same filetime than '{$sortedLogFiles[$logFileTime]}'. Unable to continue.\n"; exit(1); } else { $sortedLogFiles[$logFileTime] = $logFile; } } } ksort($sortedLogFiles); /* Read files */ echo "Read log files...\n"; foreach ($sortedLogFiles as $logFile) { global $filter; global $filterOut; $parser = new LogParser(); if ($parser->openLogFile($logFile)) { echo "File '{$logFile}' opened.\n"; while ($line = $parser->getLine()) { $logHash = $parser->formatLine($line); if (shouldBeStored($logHash["path"], $filter, $filterOut) && $logHash["status"] != '404' && $logHash["status"] != '301' && ($countSimilarRequests || !isAlreadyStored($logHash)) && $logHash["unixtime"] > $lastPiwikInsertionTime) { saveInPiwik($logHash); } } $parser->closeLogFile(); } else { echo "File '{$logFile}' is not readable.\n"; exit(1); } }
<?php /* -*- Mode: PHP; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* vim: set sw=2 ts=2 et tw=80 : */ require_once 'inc/LogParser.php'; require_once 'inc/GeneralErrorFilter.php'; require_once 'inc/ShortLogGenerator.php'; require_once 'inc/FullLogGenerator.php'; require_once 'inc/GzipUtils.php'; require_once 'inc/RunForLog.php'; require_once 'inc/Communication.php'; Headers::send(Headers::ALLOW_CROSS_ORIGIN); $run = getRequestedRun(); $logParser = new LogParser($run, new GeneralErrorFilter()); try { // Create the plain text summary too, since we need to parse the // log for errors anyway. $logParser->ensureExcerptExists(); $viewFullLog = isset($_GET['full']) && $_GET['full'] == 1; $logGenerator = $viewFullLog ? new FullLogGenerator($logParser, $run) : new ShortLogGenerator($logParser, $run); $parsedLog = $logGenerator->ensureLogExists(); GzipUtils::passThru($parsedLog, "text/html"); } catch (Exception $e) { die($e->getMessage()); }
// Log file has been rotated, read all. It is not possible on apache because server is restarted gracefully but perhaps user has done something... $data_to_parse = $new_file_size; $full = true; $return['notice'] = '<strong>' . $now . '</strong> : ' . sprintf(__('Log file has been rotated (previous size was %s and new one is %s)'), human_filesize($old_file_size), human_filesize($new_file_size)); } if ($old_file_size === 0) { $full = true; } } /* |-------------------------------------------------------------------------- | Get logs |-------------------------------------------------------------------------- | */ $logs = LogParser::getNewLines($regex, $match, $types, $tz, $max, $exclude, $file_path, $start_offset, $start_from, $load_more, $old_lastline, $multiline, $search, $data_to_parse, $full, MAX_SEARCH_LOG_TIME); /* |-------------------------------------------------------------------------- | Error while getting logs |-------------------------------------------------------------------------- | */ if (!is_array($logs)) { switch ($logs) { case '1': $return['error'] = sprintf(__('File <code>%s</code> for file ID <code>%s</code> does not exist anymore...'), $file_path, $file_id); break; default: $return['error'] = sprintf(__('Unknown error %s'), $logs); break; }
/** * */ function getUserData($log_id, $lower, $upper, $name) { //Return data if it is already loaded. if (isset($this->data)) { return $this->data; } //Initialize stuff. $data =& $this->data; $parser =& $this->parser; //Init vars. $parser = new LogParser($log_id, $lower, $upper, $this->db); $user_id = $parser->getID($name); //Start up the data class. $data = new stdClass(); $data->abilities = array('dmg' => array(), 'heal' => array(), 'taken' => array()); $data->deaths = array(); $data->totals = array('dmg' => 0, 'heal' => 0, 'taken' => 0, 'heal_modified' => 0); //Loop through the contents and prepare data. while ($status = $parser->parseLine()) { //Skip misread lines. if ($status != LogParser::SUCCESS) { continue; } //Load up a timestamp. $timestamp = $parser->getSeconds(); //Set the initial value for the timer if it hasn't started. if (!isset($data->l)) { $data->l = $timestamp; $data->timestamp = $timestamp; } //Check if this is even the person we want. if ($parser->origin_id != $user_id && $parser->target_id != $user_id || in_array($parser->attack_id, $this->global_filter)) { continue; } //Capture the data types. switch ($parser->type_id) { //Damage |3=hit,4=suffers,10=miss,15=dodge,16=parry,19=resist,23=crit| case 3: case 4: case 10: case 15: case 16: case 19: case 23: //Check if this is a player. $is_player = $parser->origin_id == $user_id; if ($is_player) { $attack =& $data->abilities['dmg'][$parser->attack_name]; } else { $attack =& $data->abilities['taken'][$parser->attack_name]; } //Set the variable if necessary. if (!isset($attack)) { $attack = array('total' => 0, 'cast' => 0, 'max' => 0, 'crits' => 0, 'misses' => 0, 'id' => $parser->attack_id); //If this isn't a player we want to add more accurate details. if (!$is_player) { $attack += array('parry' => 0, 'dodge' => 0, 'origin' => $parser->origin_name, 'block_count' => 0, 'block_total' => 0); } } $total = $is_player ? $parser->damage + $parser->absorbed : $parser->damage; $attack['total'] += $total; $attack['cast'] += 1; $attack['max'] = $attack['max'] > $total ? $attack['max'] : $total; //Calculating minimum. if (isset($attack['min'])) { $attack['min'] = $attack['min'] < $total ? $attack['min'] : $total; } else { $attack['min'] = $total; } $attack['crits'] += $parser->type_id == 23 ? 1 : 0; $attack['misses'] += in_array($parser->type_id, array(10, 15, 16, 19)) ? 1 : 0; //Log parry and dodges. if (!$is_player) { switch ($parser->type_id) { case 15: $attack['dodge']++; break; case 16: $attack['parry']++; break; } } //Log some block details. if (!$is_player && $parser->blocked > 0) { $attack['block_count']++; $attack['block_total'] += $parser->blocked; } $data->totals[$is_player ? 'dmg' : 'taken'] += $total; if (!$is_player) { //Logging a hostile action against the player. $this->logAction($parser->type_id == 23 ? 'crits' : $parser->type_id == 3 ? 'hits' : 'damages'); //We want to find the deathblow if the character is going to die. if ($parser->overkill > 0 || $this->is_dying) { $this->processDeath(); } } break; //Healing |5=heal,28=crit| //Healing |5=heal,28=crit| case 5: case 28: //Dave: For some reason disarm attacks appear in here so there's an extra statement in the if. if ($parser->origin_id == $user_id && ($parser->overheal != 0 || $parser->damage != 0)) { //Filter out stuff we don't want. if (in_array($parser->attack_id, $this->heal_filter)) { continue; } $heal =& $data->abilities['heal'][$parser->attack_name]; if (!isset($heal)) { $heal = array('total' => 0, 'total_modified' => 0, 'cast' => 0, 'max' => 0, 'min' => 0, 'crits' => 0, 'overheal' => 0, 'id' => $parser->attack_id); } $heal['total'] += $parser->damage; $heal['total_modified'] += $parser->damage + $parser->overheal; $heal['cast'] += 1; $heal['max'] = $heal['max'] > $parser->damage ? $heal['max'] : $parser->damage; $heal['min'] = isset($heal['min']) ? $heal['min'] < $parser->damage ? $heal['min'] : $parser->damage : $parser->damage; $heal['crits'] += $parser->type_id == 28 ? 1 : 0; $heal['overheal'] += $parser->overheal; $data->totals['heal'] += $parser->damage; $data->totals['heal_modified'] += $parser->damage + $parser->overheal; } else { //Logging an incoming heal to the player. $this->logAction('heals'); } break; //Deaths |11=slain,12=died| //Deaths |11=slain,12=died| case 11: case 12: if ($parser->type_id == 11 && $parser->origin_id != $user_id || $parser->type_id == 12) { $this->is_dying = true; } break; } } //Calculate the final length. $data->l = $parser->getSeconds() - $data->l; //Uset unneccesary data and sort. unset($this->parser, $this->actions); uasort($data->abilities['dmg'], array('Data_model', 'sort')); uasort($data->abilities['heal'], array('Data_model', 'sort')); uasort($data->abilities['taken'], array('Data_model', 'sort')); return $data; }
require_once 'inc/Communication.php'; Headers::send(Headers::ALLOW_CROSS_ORIGIN); $type = isset($_GET["type"]) ? $_GET["type"] : "plaintext"; $run = getRequestedRun(); try { if ($type == "reftest") { $logParser = new LogParser($run, new ReftestFailureFilter()); $reftestExcerpt = $logParser->ensureExcerptExists(); GzipUtils::passThru($reftestExcerpt, 'text/plain'); } else { if ($type == "tinderbox_print") { $logParser = new LogParser($run, new TinderboxPrintFilter()); $tinderboxPrintExcerpt = $logParser->ensureExcerptExists(); GzipUtils::passThru($tinderboxPrintExcerpt, 'text/plain'); } else { $logParser = new LogParser($run, new GeneralErrorFilter()); $rawErrorSummary = $logParser->ensureExcerptExists(); if ($type != "annotated") { GzipUtils::passThru($rawErrorSummary, 'text/plain'); } else { date_default_timezone_set('America/Los_Angeles'); $logDescription = $run['buildername'] . ' on ' . date("Y-m-d H:i:s", $run['starttime']); $annotatedSummaryGenerator = new AnnotatedSummaryGenerator($rawErrorSummary, $logDescription); $annotatedSummary = $annotatedSummaryGenerator->ensureAnnotatedSummaryExists(); GzipUtils::passThru($annotatedSummary, 'text/plain'); } } } } catch (Exception $e) { die("Log not available."); }
<?php /** * Cкрипт парсинга файлов логов */ header('Content-Type: text/html; charset=UTF-8'); require_once 'class/LogParser.class.php'; try { $lpObj = new LogParser(); $lpObj->parse(); echo 'Логи успешно сохранены в базу данных'; } catch (Exception $e) { echo $e->getMessage(); } unset($lpObj);