public function store($todelete, $outputfile) { echo "Preparing to delete [" . count($todelete) . "] trackids\n"; if (!is_dir($this->statdir)) { throw new Exception('Statistics module: output dir do not exists [' . $this->statdir . ']'); } if (!file_exists($this->inputfile)) { throw new Exception('Statistics module: input file do not exists [' . $this->inputfile . ']'); } $file = fopen($this->inputfile, 'r'); // Open the output file in a way that guarantees that we will not overwrite a random file. if (file_exists($outputfile)) { // Delete existing output file. unlink($outputfile); } $outfile = fopen($outputfile, 'x'); /* Create the output file. */ $logparser = new sspmod_statistics_LogParser($this->statconfig->getValue('datestart', 0), $this->statconfig->getValue('datelength', 15), $this->statconfig->getValue('offsetspan', 44)); $i = 0; // Parse through log file, line by line while (!feof($file)) { $logline = fgets($file, 4096); // Continue if STAT is not found on line. if (!preg_match('/STAT/', $logline)) { continue; } $i++; $content = $logparser->parseContent($logline); $action = trim($content[5]); if ($i % 10000 == 0) { echo "Read line " . $i . "\n"; } $trackid = $content[4]; if (in_array($trackid, $todelete)) { continue; } fputs($outfile, $logline); } fclose($file); fclose($outfile); }
public function aggregate($debug = FALSE) { $this->loadMetadata(); if (!is_dir($this->statdir)) { throw new Exception('Statistics module: output dir do not exists [' . $this->statdir . ']'); } if (!file_exists($this->inputfile)) { throw new Exception('Statistics module: input file do not exists [' . $this->inputfile . ']'); } $file = fopen($this->inputfile, 'r'); if ($file === FALSE) { throw new Exception('Statistics module: unable to open file [' . $this->inputfile . ']'); } $logparser = new sspmod_statistics_LogParser($this->statconfig->getValue('datestart', 0), $this->statconfig->getValue('datelength', 15), $this->statconfig->getValue('offsetspan', 44)); $datehandler = array('default' => new sspmod_statistics_DateHandler($this->offset), 'month' => new sspmod_statistics_DateHandlerMonth($this->offset)); $notBefore = 0; $lastRead = 0; $lastlinehash = '-'; if (isset($this->metadata)) { $notBefore = $this->metadata['notBefore']; $lastlinehash = $this->metadata['lastlinehash']; } $lastlogline = 'sdfsdf'; $lastlineflip = FALSE; $results = array(); $i = 0; // Parse through log file, line by line while (!feof($file)) { $logline = fgets($file, 4096); // Continue if STAT is not found on line. if (!preg_match('/STAT/', $logline)) { continue; } $i++; $lastlogline = $logline; // Parse log, and extract epoch time and rest of content. $epoch = $logparser->parseEpoch($logline); $content = $logparser->parseContent($logline); $action = trim($content[5]); if ($this->fromcmdline && $i % 10000 == 0) { echo "Read line " . $i . "\n"; } if ($debug) { echo "----------------------------------------\n"; echo 'Log line: ' . $logline . "\n"; echo 'Date parse [' . substr($logline, 0, $this->statconfig->getValue('datelength', 15)) . '] to [' . date(DATE_RFC822, $epoch) . ']' . "\n"; echo htmlentities(print_r($content, true)); if ($i >= 13) { exit; } } if ($epoch > $lastRead) { $lastRead = $epoch; } if ($epoch === $notBefore) { if (!$lastlineflip) { if (sha1($logline) === $lastlinehash) { $lastlineflip = TRUE; } continue; } } if ($epoch < $notBefore) { continue; } // Iterate all the statrules from config. foreach ($this->statrules as $rulename => $rule) { $type = 'aggregate'; if (array_key_exists('type', $rule)) { $type = $rule['type']; } if ($type !== 'aggregate') { continue; } foreach ($this->timeres as $tres => $tresconfig) { $dh = 'default'; if (isset($tresconfig['customDateHandler'])) { $dh = $tresconfig['customDateHandler']; } $timeslot = $datehandler['default']->toSlot($epoch, $tresconfig['slot']); $fileslot = $datehandler[$dh]->toSlot($epoch, $tresconfig['fileslot']); if (isset($rule['action']) && $action !== $rule['action']) { continue; } $difcol = self::getDifCol($content, $rule['col']); if (!isset($results[$rulename][$tres][$fileslot][$timeslot]['_'])) { $results[$rulename][$tres][$fileslot][$timeslot]['_'] = 0; } if (!isset($results[$rulename][$tres][$fileslot][$timeslot][$difcol])) { $results[$rulename][$tres][$fileslot][$timeslot][$difcol] = 0; } $results[$rulename][$tres][$fileslot][$timeslot]['_']++; $results[$rulename][$tres][$fileslot][$timeslot][$difcol]++; } } } $this->metadata['notBefore'] = $lastRead; $this->metadata['lastline'] = $lastlogline; $this->metadata['lastlinehash'] = sha1($lastlogline); return $results; }