/** * Parse attachments * @return array Returns array with failed or success data * (See parser-common/src/Parser.php) for more info. */ public function parse() { if ($this->arfMail !== true) { $this->feedName = 'default'; // As this is a generic FBL parser we need to see which was the source and add the name // to the report, so its origin is clearly shown. $source = $this->parsedMail->getHeader('from'); foreach (config("{$this->configBase}.parser.aliases") as $from => $alias) { if (preg_match($from, $source)) { // If there is an alias, prefer that name instead of the from address $source = $alias; // If there is an more specific feed configuration prefer that config over the default if (!empty(config("{$this->configBase}.feeds.{$source}"))) { $this->feedName = $source; } } } // If feed is known and enabled, validate data and save report if ($this->isKnownFeed() && $this->isEnabledFeed()) { // To get some more consitency, remove "\r" from the report. $this->arfMail['report'] = str_replace("\r", "", $this->arfMail['report']); // Build up the report preg_match_all("/([\\w\\-]+): (.*)[ ]*\n/m", $this->arfMail['report'], $matches); $report = array_combine($matches[1], $matches[2]); if (empty($report['Received-Date'])) { if (!empty($report['Arrival-Date'])) { $report['Received-Date'] = $report['Arrival-Date']; unset($report['Arrival-Date']); } } // Now parse the headers from the spam messages and add it to the report if (!empty($this->arfMail['evidence'])) { $spamMessage = new MimeParser(); $spamMessage->setText($this->arfMail['evidence']); $report['headers'] = $spamMessage->getHeaders(); } else { $this->failed('The e-mail received at the parser is not RFC822 compliant, and therefor not a FBL message'); } // Also add the spam message body to the report $report['body'] = $spamMessage->getMessageBody(); // Sanity check if ($this->hasRequiredFields($report) === true) { // incident has all requirements met, filter and add! $report = $this->applyFilters($report); $incident = new Incident(); $incident->source = $source; // FeedName $incident->source_id = false; $incident->ip = $report['Source-IP']; $incident->domain = false; $incident->class = config("{$this->configBase}.feeds.{$this->feedName}.class"); $incident->type = config("{$this->configBase}.feeds.{$this->feedName}.type"); $incident->timestamp = strtotime($report['Received-Date']); $incident->information = json_encode($report); $this->incidents[] = $incident; } } } return $this->success(); }
/** * Create and return a Parser class and it's configuration * @param \PhpMimeMailParser\Parser $parsedMail * @param array $arfMail * @return object parser */ public static function create($parsedMail, $arfMail) { /** * Loop through the parser list and try to find a match by * validating the send or the body according to the parsers' * configuration. */ $parsers = Factory::getParsers(); foreach ($parsers as $parserName) { $parserClass = 'AbuseIO\\Parsers\\' . $parserName; // Parser is enabled, see if we can match it's sender_map or body_map if (config("parsers.{$parserName}.parser.enabled") === true) { // Check validity of the 'report_file' setting before we continue // If no report_file is used, continue w/o validation $report_file = config("parsers.{$parserName}.parser.report_file"); if ($report_file == null || is_string($report_file) && isValidRegex($report_file)) { $isValidReport = true; } else { $isValidReport = false; Log::warning('AbuseIO\\Parsers\\Factory: ' . "The parser {$parserName} has an invalid value for 'report_file' (not a regex)."); break; } // Check the sender address foreach (config("parsers.{$parserName}.parser.sender_map") as $senderMap) { if (isValidRegex($senderMap)) { if (preg_match($senderMap, $parsedMail->getHeader('from')) && $isValidReport) { return new $parserClass($parsedMail, $arfMail); } } else { Log::warning('AbuseIO\\Parsers\\Factory: ' . "The parser {$parserName} has an invalid value for 'sender_map' (not a regex)."); } } // If no valid sender is found, check the body foreach (config("parsers.{$parserName}.parser.body_map") as $bodyMap) { if (isValidRegex($bodyMap)) { if (preg_match($bodyMap, $parsedMail->getMessageBody()) && $isValidReport) { return new $parserClass($parsedMail, $arfMail); } if ($arfMail !== false) { foreach ($arfMail as $mailPart) { if (preg_match($bodyMap, $mailPart)) { return new $parserClass($parsedMail, $arfMail); } } } } else { Log::warning('AbuseIO\\Parsers\\Factory: ' . "The parser {$parserName} has an invalid value for 'body_map' (not a regex)."); } } } else { Log::info('AbuseIO\\Parsers\\Factory: ' . "The parser {$parserName} has been disabled and will not be used for this message."); } } // No valid parsers found return false; }
/** * Parse the given email resource * * @param string $file path/to/file * @return Parser */ public function parseFile($file) { if (!file_exists($file)) { throw new Exception("File {$file} does not exist."); } $this->lines = file($file); $this->parser->setPath($file); $bounceReason = $this->findBounceReason(); fputcsv($this->csv, array($this->findRecipient(), key($bounceReason), current($bounceReason)), $this->csvDelimiter, $this->csvEnclosure); return $this; }
public function create(string $mailbox, Horde_Imap_Client_Data_Fetch $hordeEmail) : Email { // Parse the message body $parser = new Parser(); $parser->setText($hordeEmail->getFullMsg()); $htmlContent = utf8_encode((string) $parser->getMessageBody('html')); $textContent = utf8_encode((string) $parser->getMessageBody('text')); // Filter HTML body to have only safe HTML $htmlContent = trim($this->htmlFilter->purify($htmlContent)); // If no HTML content, use the text content if ($htmlContent == '') { $htmlContent = nl2br($textContent); } // The envelope contains the headers $envelope = $hordeEmail->getEnvelope(); $from = []; foreach ($envelope->from as $hordeFrom) { /** @var Horde_Mail_Rfc822_Address $hordeFrom */ if ($hordeFrom->bare_address) { $from[] = new EmailAddress($hordeFrom->bare_address, $hordeFrom->personal); } } $to = []; foreach ($envelope->to as $hordeTo) { /** @var Horde_Mail_Rfc822_Address $hordeTo */ if ($hordeTo->bare_address) { $to[] = new EmailAddress($hordeTo->bare_address, $hordeTo->personal); } } $messageId = $this->parseMessageId($envelope->message_id); $inReplyTo = $this->parseMessageId($envelope->in_reply_to); $message = new Email((string) $hordeEmail->getUid(), $messageId, $mailbox, $envelope->subject, $htmlContent, $textContent, $from, $to, $inReplyTo); $date = new DateTime(); $date->setTimestamp($envelope->date->getTimestamp()); $message->setDate($date); $flags = $hordeEmail->getFlags(); if (in_array(Horde_Imap_Client::FLAG_SEEN, $flags)) { $message->setRead(true); } return $message; }
/** * @param $rawMessage * * @return \MailChecker\Models\Message */ protected function getMessage($rawMessage) { $parser = new MailParser(); $parser->setText($rawMessage); $headers = array_change_key_case($parser->getHeaders(), CASE_LOWER); $message = new Message(); try { $message->setDate(new \DateTime($headers['date'])); } catch (\Exception $e) { // Can't recognize date time format // TODO add config option for date time format parsing $message->setDate(new \DateTime()); } $message->setSubject($headers['subject']); $message->setFrom($headers['from']); $message->setTo($headers['to']); if (isset($headers['cc'])) { $message->setCc($headers['cc']); } foreach ($parser->getParts() as $part) { if (in_array($part['content-type'], ['text/plain', 'text/html'])) { $body = new Body(); $body->setContentType($part['content-type']); $body->setCharset(isset($part['content-charset']) ? $part['content-charset'] : null); $body->setEncoding(isset($part['transfer-encoding']) ? $part['transfer-encoding'] : null); $start = $part['starting-pos-body']; $end = $part['ending-pos-body']; $body->setBody(substr($rawMessage, $start, $end - $start)); $message->addBody($body); } } /** @var \PhpMimeMailParser\Attachment $messageAttachment */ foreach ($parser->getAttachments() as $messageAttachment) { $attachment = new Attachment(); $attachment->setType($messageAttachment->getContentType()); $attachment->setFilename($messageAttachment->getFilename()); $message->addAttachment($attachment); } return $message; }
/** * @dataProvider provideData */ public function testFromStream($mid, $subjectExpected, $fromExpected, $toExpected, $textExpected, $htmlExpected, $attachmentsExpected, $countEmbeddedExpected) { //Init $file = __DIR__ . '/mails/' . $mid; $attach_dir = __DIR__ . '/mails/attach_' . $mid . '/'; //Load From Path $Parser = new Parser(); $Parser->setStream(fopen($file, 'r')); //Test Header : subject $this->assertEquals($subjectExpected, $Parser->getHeader('subject')); $this->assertArrayHasKey('subject', $Parser->getHeaders()); //Test Header : from $this->assertEquals($fromExpected, $Parser->getHeader('from')); $this->assertArrayHasKey('from', $Parser->getHeaders()); //Test Header : to $this->assertEquals($toExpected, $Parser->getHeader('to')); $this->assertArrayHasKey('to', $Parser->getHeaders()); //Test Invalid Header $this->assertFalse($Parser->getHeader('azerty')); $this->assertArrayNotHasKey('azerty', $Parser->getHeaders()); //Test Body : text if ($textExpected[0] == 'COUNT') { $this->assertEquals($textExpected[1], substr_count($Parser->getMessageBody('text'), $textExpected[2])); } elseif ($textExpected[0] == 'MATCH') { $this->assertEquals($textExpected[1], $Parser->getMessageBody('text')); } //Test Body : html if ($htmlExpected[0] == 'COUNT') { $this->assertEquals($htmlExpected[1], substr_count($Parser->getMessageBody('html'), $htmlExpected[2])); } elseif ($htmlExpected[0] == 'MATCH') { $this->assertEquals($htmlExpected[1], $Parser->getMessageBody('html')); } //Test Nb Attachments $attachments = $Parser->getAttachments(); $this->assertEquals(count($attachmentsExpected), count($attachments)); $iterAttachments = 0; //Test Attachments $attachmentsEmbeddedToCheck = array(); if (count($attachmentsExpected) > 0) { //Save attachments $Parser->saveAttachments($attach_dir); foreach ($attachmentsExpected as $attachmentExpected) { //Test Exist Attachment $this->assertTrue(file_exists($attach_dir . $attachmentExpected[0])); //Test Filename Attachment $this->assertEquals($attachmentExpected[0], $attachments[$iterAttachments]->getFilename()); //Test Size Attachment $this->assertEquals($attachmentExpected[1], filesize($attach_dir . $attachments[$iterAttachments]->getFilename())); //Test Inside Attachment if ($attachmentExpected[2] != '' && $attachmentExpected[3] > 0) { $fileContent = file_get_contents($attach_dir . $attachments[$iterAttachments]->getFilename(), FILE_USE_INCLUDE_PATH); $this->assertEquals($attachmentExpected[3], substr_count($fileContent, $attachmentExpected[2])); $this->assertEquals($attachmentExpected[3], substr_count($attachments[$iterAttachments]->getContent(), $attachmentExpected[2])); } //Test ContentType Attachment $this->assertEquals($attachmentExpected[4], $attachments[$iterAttachments]->getContentType()); //Test ContentDisposition Attachment $this->assertEquals($attachmentExpected[5], $attachments[$iterAttachments]->getContentDisposition()); //Test md5 of Headers Attachment $this->assertEquals($attachmentExpected[6], md5(serialize($attachments[$iterAttachments]->getHeaders()))); //Save embedded Attachments to check if ($attachmentExpected[7] != '') { array_push($attachmentsEmbeddedToCheck, $attachmentExpected[7]); } //Remove Attachment unlink($attach_dir . $attachments[$iterAttachments]->getFilename()); $iterAttachments++; } //Remove Attachment Directory rmdir($attach_dir); } else { $this->assertFalse($Parser->saveAttachments($attach_dir)); } //Test embedded Attachments $htmlEmbedded = $Parser->getMessageBody('htmlEmbedded'); $this->assertEquals($countEmbeddedExpected, substr_count($htmlEmbedded, "data:")); if (!empty($attachmentsEmbeddedToCheck)) { foreach ($attachmentsEmbeddedToCheck as $itemExpected) { $this->assertEquals(1, substr_count($htmlEmbedded, $itemExpected)); } } }
/** * @param $filename * * @return bool */ public function getAttachment($filename) { $data = Storage::get($this->filename); $email = new MimeParser(); $email->setText($data); foreach ($email->getAttachments() as $attachment) { if ($attachment->getFilename() == $filename) { return $attachment; } } return false; }
/** * Shows the evidence for this ticket as webpage. * * @param Ticket $ticket Ticket Model * @param integer $evidenceId * @return \Illuminate\Http\Response */ public function viewEvidence(Ticket $ticket, $evidenceId) { $evidence = Evidence::find($evidenceId); if (!$evidence) { return Redirect::route('admin.tickets.show', $ticket->id)->with('message', 'The evidence is no longer available for this event.'); } if (is_file($evidence->filename)) { $evidenceData = file_get_contents($evidence->filename); $evidenceParsed = new MimeParser(); $evidenceParsed->setText($evidenceData); $filesystem = new Filesystem(); $evidenceTempDir = "/tmp/abuseio/cache/{$ticket->id}/{$evidenceId}/"; if (!$filesystem->isDirectory($evidenceTempDir)) { if (!$filesystem->makeDirectory($evidenceTempDir, 0755, true)) { Log::error(get_class($this) . ': ' . 'Unable to create temp directory: ' . $evidenceTempDir); } $evidenceParsed->saveAttachments($evidenceTempDir); } return view('tickets.evidence')->with('ticket', $ticket)->with('evidence', $evidenceParsed)->with('evidenceId', $evidenceId)->with('evidenceTempDir', $evidenceTempDir)->with('auth_user', $this->auth_user); } else { return Redirect::route('admin.tickets.show', $ticket->id)->with('message', 'ERROR: The file was not available on the filesystem.'); } }
/** * Execute the command * * @return void */ public function handle() { Log::info('(JOB ' . getmypid() . ') ' . get_class($this) . ': ' . 'Queued worker is starting the processing of email file: ' . $this->filename); $filesystem = new Filesystem(); $rawEmail = $filesystem->get($this->filename); $parsedMail = new MimeParser(); $parsedMail->setText($rawEmail); // Sanity checks if (empty($parsedMail->getHeader('from')) || empty($parsedMail->getMessageBody())) { Log::warning('(JOB ' . getmypid() . ') ' . get_class($this) . ': ' . 'Missing e-mail headers from and/or empty body: ' . $this->filename); $this->alertAdmin(); return; } // Ignore email from our own notification address to prevent mail loops if (preg_match('/' . Config::get('main.notifications.from_address') . '/', $parsedMail->getHeader('from'))) { Log::warning('(JOB ' . getmypid() . ') ' . get_class($this) . ': ' . 'Loop prevention: Ignoring email from self ' . Config::get('main.notifications.from_address')); $this->alertAdmin(); return; } // Start with detecting valid ARF e-mail $attachments = $parsedMail->getAttachments(); $arfMail = []; foreach ($attachments as $attachment) { if ($attachment->contentType == 'message/feedback-report') { $arfMail['report'] = $attachment->getContent(); } if ($attachment->contentType == 'message/rfc822') { $arfMail['evidence'] = utf8_encode($attachment->getContent()); } if ($attachment->contentType == 'text/plain') { $arfMail['message'] = $attachment->getContent(); } } /* * Sometimes the mime header does not set the main message correctly. This is ment as a fallback and will * use the original content body (which is basicly the same mime element). But only fallback if we actually * have a RFC822 message with a feedback report. */ if (empty($arfMail['message']) && isset($arfMail['report']) && isset($arfMail['evidence'])) { $arfMail['message'] = $parsedMail->getMessageBody(); } // If we do not have a complete e-mail, then we empty the perhaps partially filled arfMail // which is useless, hence reset to false if (!isset($arfMail['report']) || !isset($arfMail['evidence']) || !isset($arfMail['message'])) { $arfMail = false; } // Asking ParserFactory for an object based on mappings, or die trying $parser = ParserFactory::create($parsedMail, $arfMail); if ($parser !== false) { $parserResult = $parser->parse(); } else { Log::error('(JOB ' . getmypid() . ') ' . get_class($this) . ': ' . ': No parser available to handle message from : ' . $parsedMail->getHeader('from') . ' with subject: ' . $parsedMail->getHeader('subject')); $this->alertAdmin(); return; } if ($parserResult !== false && $parserResult['errorStatus'] === true) { Log::error('(JOB ' . getmypid() . ') ' . get_class($parser) . ': ' . ': Parser has ended with fatal errors ! : ' . $parserResult['errorMessage']); $this->alertAdmin(); return; } else { Log::info('(JOB ' . getmypid() . ') ' . get_class($parser) . ': ' . ': Parser completed with ' . $parserResult['warningCount'] . ' warnings and collected ' . count($parserResult['data']) . ' events to save'); } if ($parserResult['warningCount'] !== 0 && Config::get('main.emailparser.notify_on_warnings') === true) { Log::error('(JOB ' . getmypid() . ') ' . get_class($this) . ': ' . 'Configuration has warnings set as critical and ' . $parserResult['warningCount'] . ' warnings were detected. Sending alert to administrator'); $this->alertAdmin(); return; } if (count($parserResult['data']) !== 0) { // Call validator $validator = new EventsValidate(); $validatorResult = $validator->check($parserResult['data']); if ($validatorResult['errorStatus'] === true) { Log::error('(JOB ' . getmypid() . ') ' . get_class($validator) . ': ' . 'Validator has ended with errors ! : ' . $validatorResult['errorMessage']); $this->alertAdmin(); return; } else { Log::info('(JOB ' . getmypid() . ') ' . get_class($validator) . ': ' . 'Validator has ended without errors'); } /** * save evidence into table **/ $evidence = new Evidence(); $evidence->filename = $this->filename; $evidence->sender = $parsedMail->getHeader('from'); $evidence->subject = $parsedMail->getHeader('subject'); $evidence->save(); /** * call saver **/ $saver = new EventsSave(); $saverResult = $saver->save($parserResult['data'], $evidence->id); /** * We've hit a snag, so we are gracefully killing ourselves * after we contact the admin about it. EventsSave should never * end with problems unless the mysql died while doing transactions **/ if ($saverResult['errorStatus'] === true) { Log::error('(JOB ' . getmypid() . ') ' . get_class($saver) . ': ' . 'Saver has ended with errors ! : ' . $saverResult['errorMessage']); $this->alertAdmin(); return; } else { Log::info('(JOB ' . getmypid() . ') ' . get_class($saver) . ': ' . 'Saver has ended without errors'); } } else { Log::warning('(JOB ' . getmypid() . ') ' . get_class($this) . ': ' . 'Parser did not return any events therefore skipping validation and saving a empty event set'); } Log::info('(JOB ' . getmypid() . ') ' . get_class($this) . ': ' . 'Queued worker has ended the processing of email file: ' . $this->filename); }
/** * Execute the command. * * @return void */ public function handle() { Log::info(get_class($this) . ': Queued worker is starting the processing of email file: ' . $this->filename); $filesystem = new Filesystem(); $rawEmail = $filesystem->get($this->filename); $parsedMail = new MimeParser(); $parsedMail->setText($rawEmail); // Sanity checks if (empty($parsedMail->getHeader('from')) || empty($parsedMail->getMessageBody())) { Log::warning(get_class($this) . 'Validation failed on: ' . $this->filename); $this->exception(); } // Ignore email from our own notification address to prevent mail loops if (preg_match('/' . Config::get('main.notifications.from_address') . '/', $parsedMail->getHeader('from'))) { Log::warning(get_class($this) . 'Loop prevention: Ignoring email from self ' . Config::get('main.notifications.from_address')); $this->exception(); } // Start with detecting valid ARF e-mail $attachments = $parsedMail->getAttachments(); $arfMail = []; foreach ($attachments as $attachment) { if ($attachment->contentType == 'message/feedback-report') { $arfMail['report'] = $attachment->getContent(); } if ($attachment->contentType == 'message/rfc822') { $arfMail['evidence'] = $attachment->getContent(); } if ($attachment->contentType == 'text/plain') { $arfMail['message'] = $attachment->getContent(); } } // If we do not have a complete e-mail, then we empty the perhaps partially filled arfMail // which is useless, hence reset to false if (!isset($arfMail['report']) || !isset($arfMail['evidence']) || !isset($arfMail['message'])) { $arfMail = false; } // Asking GetParser for an object based on mappings, or die trying $parser = GetParser::object($parsedMail, $arfMail); $result = false; $events = false; if ($parser !== false) { $result = $parser->parse(); } else { Log::error(get_class($this) . ': Unable to handle message from: ' . $parsedMail->getHeader('from') . ' with subject: ' . $parsedMail->getHeader('subject')); $this->exception(); } if ($result !== false && $result['errorStatus'] !== true) { Log::info(get_class($parser) . ': Parser as ended without errors. Collected ' . count($result['data']) . ' events to save'); $events = $result['data']; } else { Log::error(get_class($parser) . ': Parser as ended with errors ! : ' . $result['errorMessage']); $this->exception(); } // call validater $validator = new EventsValidate($events); $return = $validator->handle(); if ($return['errorStatus'] === false) { Log::error(get_class($validator) . ': Validator as ended with errors ! : ' . $result['errorMessage']); $this->exception(); } else { Log::info(get_class($validator) . ': Validator as ended without errors'); } // save evidence into table $evidence = new Evidence(); $evidence->filename = $this->filename; $evidence->sender = $parsedMail->getHeader('from'); $evidence->subject = $parsedMail->getHeader('subject'); $evidence->save(); // call saver $saver = new EventsSave($events, $evidence->id); $return = $saver->handle(); if ($return['errorStatus'] === false) { Log::error(get_class($saver) . ': Saver as ended with errors ! : ' . $result['errorMessage']); $this->exception(); } else { Log::info(get_class($saver) . ': Saver as ended without errors'); } }
/** * Execute the console command. * * @return bool */ public function handle() { $account = Account::getSystemAccount(); if (empty($this->option('start')) && empty($this->option('prepare')) && empty($this->option('clean'))) { $this->error('You need to either prepare or start the migration. try --help'); die; } if (!empty($this->option('clean'))) { $this->warn('This will remove all data from the database except prepared data. Do not run ' . 'this in producion and only when a migration has gone bad and you want to restart it'); if ($this->confirm('Do you wish to continue? [y|N]')) { $this->info('starting clean up'); DB::statement('SET foreign_key_checks=0'); Note::truncate(); Event::truncate(); Ticket::truncate(); Netblock::truncate(); Contact::truncate(); DB::statement('SET foreign_key_checks=1'); } else { $this->info('cancelled clean up'); } } /* * Combine the use of start/stop and threading here for usage with prepare and start of migration */ $startFrom = $this->option('startfrom'); $endWith = $this->option('endwith'); if (!empty($this->option('prepare')) || !empty($this->option('start'))) { if (!empty($this->option('threaded'))) { if (empty($this->option('threadid')) || empty($this->option('threadsize'))) { $this->error('threadid and threadsize are required to calculate this threads start/stop ID'); die; } $this->info("*** using threaded mode, instance ID {$this->option('threadid')}"); $startFrom = $this->option('threadid') * $this->option('threadsize') - $this->option('threadsize') + 1; $endWith = $startFrom + $this->option('threadsize') - 1; $this->info("*** starting with ticket {$startFrom} " . "and ending with ticket {$endWith} "); } } if (!empty($this->option('prepare'))) { $this->info('building required evidence cache files'); $filesystem = new Filesystem(); $path = storage_path() . '/migration/'; umask(07); if (!$filesystem->isDirectory($path)) { // If a datefolder does not exist, then create it or die trying if (!$filesystem->makeDirectory($path, 0770)) { $this->error('Unable to create directory: ' . $path); $this->exception(); } if (!is_dir($path)) { $this->error('Path vanished after write: ' . $path); $this->exception(); } chgrp($path, config('app.group')); } DB::setDefaultConnection('abuseio3'); $evidenceRows = DB::table('Evidence')->where('id', '>=', $startFrom)->where('id', '<=', $endWith); $migrateCount = $evidenceRows->count(); $evidences = $evidenceRows->get(); $this->output->progressStart($migrateCount); // If there are now rows to do, advance to complete if ($migrateCount === 0) { $this->output->progressAdvance(); echo ' nothing to do, because there are no records in this selection'; } foreach ($evidences as $evidence) { $this->output->progressAdvance(); // Before we go into an error, lets see if this evidence was even linked to any ticket at all // If not we can ignore the error and just smile and wave $evidenceLinks = DB::table('EvidenceLinks')->where('EvidenceID', '=', $evidence->ID)->get(); if (count($evidenceLinks) === 0) { echo " skipping unlinked evidence ID {$evidence->ID} "; continue; } $filename = $path . "evidence_id_{$evidence->ID}.data"; if (is_file($filename)) { // Check weither the evidence was actually created in the database DB::setDefaultConnection('mysql'); $evidenceCheck = Evidence::where('id', $evidence->ID); if ($evidenceCheck->count() !== 1) { $this->error("file {$filename} exists however not in database. try deleting it to retry"); die; } DB::setDefaultConnection('abuseio3'); continue; } else { echo " working on evidence ID {$evidence->ID} "; } $rawEmail = $evidence->Data; $parsedMail = new MimeParser(); $parsedMail->setText($rawEmail); // Start with detecting valid ARF e-mail $attachments = $parsedMail->getAttachments(); $arfMail = []; foreach ($attachments as $attachment) { if ($attachment->contentType == 'message/feedback-report') { $arfMail['report'] = $attachment->getContent(); } if ($attachment->contentType == 'message/rfc822') { $arfMail['evidence'] = utf8_encode($attachment->getContent()); } if ($attachment->contentType == 'text/plain') { $arfMail['message'] = $attachment->getContent(); } } if (empty($arfMail['message']) && isset($arfMail['report']) && isset($arfMail['evidence'])) { $arfMail['message'] = $parsedMail->getMessageBody(); } // If we do not have a complete e-mail, then we empty the perhaps partially filled arfMail // which is useless, hence reset to false if (!isset($arfMail['report']) || !isset($arfMail['evidence']) || !isset($arfMail['message'])) { $arfMail = false; } // Asking ParserFactory for an object based on mappings, or die trying $parser = ParserFactory::create($parsedMail, $arfMail); if ($parser !== false) { $parserResult = $parser->parse(); } else { $this->error('No parser available to handle message ' . $evidence->ID . ' from : ' . $evidence->Sender . ' with subject: ' . $evidence->Subject); continue; } if ($parserResult !== false && $parserResult['errorStatus'] === true) { $this->error('Parser has ended with fatal errors ! : ' . $parserResult['errorMessage']); $this->exception(); } if ($parserResult['warningCount'] !== 0) { $this->error('Configuration has warnings set as critical and ' . $parserResult['warningCount'] . ' warnings were detected.'); //var_dump($rawEmail); $this->exception(); } // Write the evidence into the archive $evidenceWrite = new EvidenceSave(); $evidenceData = $rawEmail; $evidenceFile = $evidenceWrite->save($evidenceData); // Save the file reference into the database $evidenceSave = new Evidence(); $evidenceSave->id = $evidence->ID; $evidenceSave->filename = $evidenceFile; $evidenceSave->sender = $parsedMail->getHeader('from'); $evidenceSave->subject = $parsedMail->getHeader('subject'); $incidentsProcess = new IncidentsProcess($parserResult['data'], $evidenceSave); /* * Because google finds it 'obvious' not to include the IP address relating to abuse * the IP field might now be empty with reparsing if the domain/label does not resolve * anymore. For these cases we need to lookup the ticket that was linked to the evidence * match the domain and retrieve its IP. */ foreach ($parserResult['data'] as $index => $incident) { if ($incident->source == 'Google Safe Browsing' && $incident->domain != false && $incident->ip == '127.0.0.1') { // Get the list of tickets related to this evidence $evidenceLinks = DB::table('EvidenceLinks')->where('EvidenceID', '=', $evidence->ID)->get(); // For each ticket check if the domain name is matching the evidence we need to update foreach ($evidenceLinks as $evidenceLink) { $ticket = DB::table('Reports')->where('ID', '=', $evidenceLink->ReportID)->first(); if ($ticket->Domain == $incident->domain) { $incident->ip = $ticket->IP; } } // Update the original object by overwriting it $parserResult['data'][$index] = $incident; } } // Only continue if not empty, empty set is acceptable (exit OK) if (!$incidentsProcess->notEmpty()) { $this->warn("No evidence build, no results from parser for {$evidence->ID}"); continue; } // Validate the data set if (!$incidentsProcess->validate()) { $this->error('Validation failed of object.'); $this->exception(); } $incidents = []; foreach ($parserResult['data'] as $incident) { $incidents[$incident->ip][] = $incident; } DB::setDefaultConnection('mysql'); $evidenceSave->save(); DB::setDefaultConnection('abuseio3'); $output = ['evidenceId' => $evidence->ID, 'evidenceData' => $evidence->Data, 'incidents' => $incidents, 'newId' => $evidenceSave->id]; if ($filesystem->put($filename, json_encode($output)) === false) { $this->error('Unable to write file: ' . $filename); return false; } } $this->output->progressFinish(); } if (!empty($this->option('start'))) { if (empty($this->option('skipcontacts'))) { $this->info('starting migration - phase 1 - contact data'); DB::setDefaultConnection('abuseio3'); $customers = DB::table('Customers')->get(); DB::setDefaultConnection('mysql'); $this->output->progressStart(count($customers)); foreach ($customers as $customer) { $newContact = new Contact(); $newContact->reference = $customer->Code; $newContact->name = $customer->Name; $newContact->email = $customer->Contact; $newContact->auto_notify = $customer->AutoNotify; $newContact->enabled = 1; $newContact->account_id = $account->id; $newContact->created_at = Carbon::parse($customer->LastModified); $newContact->updated_at = Carbon::parse($customer->LastModified); $validation = Validator::make($newContact->toArray(), Contact::createRules()); if ($validation->fails()) { $message = implode(' ', $validation->messages()->all()); $this->error('fatal error while creating contacts :' . $message); $this->exception(); } else { $newContact->save(); } $this->output->progressAdvance(); echo " Working on contact {$customer->Code} "; } $this->output->progressFinish(); } else { $this->info('skipping migration - phase 1 - contact data'); } if (empty($this->option('skipnetblocks'))) { $this->info('starting migration - phase 2 - netblock data'); DB::setDefaultConnection('abuseio3'); $netblocks = DB::table('Netblocks')->get(); DB::setDefaultConnection('mysql'); $this->output->progressStart(count($netblocks)); foreach ($netblocks as $netblock) { $contact = FindContact::byId($netblock->CustomerCode); if ($contact->reference != $netblock->CustomerCode) { $this->error('Contact lookup failed, mismatched results'); $this->{$this}->exception(); } $newNetblock = new Netblock(); $newNetblock->first_ip = long2ip($netblock->begin_in); $newNetblock->last_ip = long2ip($netblock->end_in); $newNetblock->description = 'Imported from previous AbuseIO version which did not include a description'; $newNetblock->contact_id = $contact->id; $newNetblock->enabled = 1; $newNetblock->created_at = Carbon::parse($netblock->LastModified); $newNetblock->updated_at = Carbon::parse($netblock->LastModified); $validation = Validator::make($newNetblock->toArray(), Netblock::createRules($newNetblock)); if ($validation->fails()) { $message = implode(' ', $validation->messages()->all()); $this->error('fatal error while creating contacts :' . $message); $this->exception(); } else { $newNetblock->save(); } $this->output->progressAdvance(); echo ' Working on netblock ' . long2ip($netblock->begin_in) . ' '; } $this->output->progressFinish(); } else { $this->info('skipping migration - phase 2 - netblock data'); } if (empty($this->option('skiptickets'))) { $this->info('starting migration - phase 3 - ticket and evidence data'); DB::setDefaultConnection('abuseio3'); $ticketRows = DB::table('Reports')->where('id', '>=', $startFrom)->where('id', '<=', $endWith); $migrateCount = $ticketRows->count(); $tickets = $ticketRows->get(); DB::setDefaultConnection('mysql'); $this->output->progressStart($migrateCount); // If there are now rows to do, advance to complete if ($migrateCount === 0) { $this->output->progressAdvance(); echo ' nothing to do, because there are no records in this selection'; } foreach ($tickets as $ticket) { // Get the list of evidence ID's related to this ticket DB::setDefaultConnection('abuseio3'); $evidenceLinks = DB::table('EvidenceLinks')->where('ReportID', '=', $ticket->ID)->get(); DB::setDefaultConnection('mysql'); // DO NOT REMOVE! Legacy versions (1.0 / 2.0) have imports without evidence. // These dont have any linked evidence and will require a manual building of evidence // for now we ignore them. This will not affect any 3.x installations if ($ticket->CustomerName == 'Imported from AbuseReporter' || !empty(json_decode($ticket->Information)->importnote)) { // Manually build the evidence $this->output->progressAdvance(); echo " Working on events from ticket {$ticket->ID}"; $this->replayTicket($ticket, $account); continue; } if (count($evidenceLinks) != (int) $ticket->ReportCount) { // Count does not match, known 3.0 limitation related to not always saving all the data // so we will do a little magic to fix that $this->output->progressAdvance(); echo " Working on events from ticket {$ticket->ID}"; $this->replayTicket($ticket, $account); continue; } else { // Just work as normal $this->output->progressAdvance(); echo " Working on events from ticket {$ticket->ID}"; $newTicket = $this->createTicket($ticket, $account); // Create all the events foreach ($evidenceLinks as $evidenceLink) { $path = storage_path() . '/migration/'; $filename = $path . "evidence_id_{$evidenceLink->EvidenceID}.data"; if (!is_file($filename)) { $this->error('missing cache file '); $this->exception(); } $evidence = json_decode(file_get_contents($filename)); $evidenceID = (int) $evidence->evidenceId; $incidents = $evidence->incidents; // Yes we only grab nr 0 from the array, because that is what the old aggregator did // which basicly ignored a few incidents because they werent considered unique (which // they were with the domain name) $ip = $newTicket->ip; if (property_exists($incidents, $ip)) { $incidentTmp = $incidents->{$ip}; $incident = $incidentTmp[0]; $newEvent = new Event(); $newEvent->evidence_id = $evidenceID; $newEvent->information = $incident->information; $newEvent->source = $incident->source; $newEvent->ticket_id = $newTicket->id; $newEvent->timestamp = $incident->timestamp; } else { // Parser did not find any related evidence so replay it from the ticket only reason // Why it happends here if DNS has changed and google report cannot be matched $newEvent = new Event(); $newEvent->evidence_id = $evidenceID; $newEvent->information = $ticket->Information; $newEvent->source = $ticket->Source; $newEvent->ticket_id = $newTicket->id; $newEvent->timestamp = $ticket->FirstSeen; } // Validate the model before saving $validator = Validator::make(json_decode(json_encode($newEvent), true), Event::createRules()); if ($validator->fails()) { $this->error('DevError: Internal validation failed when saving the Event object ' . implode(' ', $validator->messages()->all())); $this->exception(); } $newEvent->save(); } } } $this->output->progressFinish(); } else { $this->info('skipping migration - phase 3 - Tickets'); } if (empty($this->option('skipnotes'))) { $this->info('starting migration - phase 4 - Notes'); DB::setDefaultConnection('abuseio3'); $notes = DB::table('Notes')->get(); DB::setDefaultConnection('mysql'); $this->output->progressStart(count($notes)); foreach ($notes as $note) { $newNote = new Note(); $newNote->id = $note->ID; $newNote->ticket_id = $note->ReportID; $newNote->submitter = $note->Submittor; $newNote->text = $note->Text; $newNote->hidden = true; $newNote->viewed = true; $newNote->created_at = Carbon::parse($note->LastModified); $newNote->updated_at = Carbon::parse($note->LastModified); $validation = Validator::make($newNote->toArray(), Note::createRules()); if ($validation->fails()) { $message = implode(' ', $validation->messages()->all()); $this->error('fatal error while creating contacts :' . $message); $this->exception(); } else { $newNote->save(); } $this->output->progressAdvance(); echo " Working on note {$note->ID} "; } $this->output->progressFinish(); } else { $this->info('skipping migration - phase 4 - Notes'); } } return true; }
/** * Execute the command. * * @return void */ public function handle() { Log::info(get_class($this) . ': ' . 'Queued worker is starting the processing of email file: ' . $this->filename); $rawEmail = Storage::get($this->filename); $parsedMail = new MimeParser(); $parsedMail->setText($rawEmail); // Sanity checks if (empty($parsedMail->getHeader('from')) || empty($parsedMail->getMessageBody())) { Log::warning(get_class($this) . ': ' . 'Missing e-mail headers from and/or empty body: ' . $this->filename); $this->exception(); return; } // Ignore email from our own notification address to prevent mail loops if (preg_match('/' . Config::get('main.notifications.from_address') . '/', $parsedMail->getHeader('from'))) { Log::warning(get_class($this) . ': ' . 'Loop prevention: Ignoring email from self ' . Config::get('main.notifications.from_address')); $this->exception(); return; } // Start with detecting valid ARF e-mail $attachments = $parsedMail->getAttachments(); $arfMail = []; foreach ($attachments as $attachment) { if ($attachment->contentType == 'message/feedback-report') { $arfMail['report'] = $attachment->getContent(); } if ($attachment->contentType == 'message/rfc822') { $arfMail['evidence'] = utf8_encode($attachment->getContent()); } if ($attachment->contentType == 'text/plain') { $arfMail['message'] = $attachment->getContent(); } } /* * Sometimes the mime header does not set the main message correctly. This is ment as a fallback and will * use the original content body (which is basicly the same mime element). But only fallback if we actually * have a RFC822 message with a feedback report. */ if (empty($arfMail['message']) && isset($arfMail['report']) && isset($arfMail['evidence'])) { $arfMail['message'] = $parsedMail->getMessageBody(); } // If we do not have a complete e-mail, then we empty the perhaps partially filled arfMail // which is useless, hence reset to false if (!isset($arfMail['report']) || !isset($arfMail['evidence']) || !isset($arfMail['message'])) { $arfMail = false; } // Asking ParserFactory for an object based on mappings, or die trying $parser = ParserFactory::create($parsedMail, $arfMail); if ($parser !== false) { $parserResult = $parser->parse(); } else { Log::error(get_class($this) . ': ' . ': No parser available to handle message from : ' . $parsedMail->getHeader('from') . ' with subject: ' . $parsedMail->getHeader('subject')); $this->exception(); return; } if ($parserResult !== false && $parserResult['errorStatus'] === true) { Log::error(get_class($parser) . ': ' . ': Parser has ended with fatal errors ! : ' . $parserResult['errorMessage']); $this->exception(); return; } else { Log::info(get_class($parser) . ': ' . ': Parser completed with ' . $parserResult['warningCount'] . ' warnings and collected ' . count($parserResult['data']) . ' incidents to save'); } if ($parserResult['warningCount'] !== 0 && Config::get('main.emailparser.notify_on_warnings') === true) { Log::error(get_class($this) . ': ' . 'Configuration has warnings set as critical and ' . $parserResult['warningCount'] . ' warnings were detected. Sending alert to administrator'); $this->exception(); return; } /* * build evidence model, but wait with saving it **/ $evidence = new Evidence(); $evidence->filename = $this->filename; $evidence->sender = $parsedMail->getHeader('from'); $evidence->subject = $parsedMail->getHeader('subject'); /* * Call IncidentsProcess to validate, store evidence and save incidents */ $incidentsProcess = new IncidentsProcess($parserResult['data'], $evidence); // Only continue if not empty, empty set is acceptable (exit OK) if (!$incidentsProcess->notEmpty()) { return; } // Validate the data set if (!$incidentsProcess->validate()) { $this->exception(); return; } // Write the data set to database if (!$incidentsProcess->save()) { $this->exception(); return; } Log::info(get_class($this) . ': ' . 'Queued worker has ended the processing of email file: ' . $this->filename); }
/** * This is a ARF mail with a single incident * * @return array $reports */ public function parseSpamReportArf() { $reports = []; //Seriously spamcop? Newlines arent in the CL specifications $this->arfMail['report'] = str_replace("\r", "", $this->arfMail['report']); preg_match_all('/([\\w\\-]+): (.*)[ ]*\\r?\\n/', $this->arfMail['report'], $regs); $report = array_combine($regs[1], $regs[2]); //Valueable information put in the body instead of the report, thnx for that Spamcop if (strpos($this->arfMail['message'], 'Comments from recipient') !== false) { preg_match("/Comments from recipient.*\\s]\n(.*)\n\n\nThis/s", str_replace(array("\r", "> "), "", $this->arfMail['message']), $match); $report['recipient_comment'] = str_replace("\n", " ", $match[1]); } // Add the headers from evidence into infoblob $parsedEvidence = new MimeParser(); $parsedEvidence->setText($this->arfMail['evidence']); $headers = $parsedEvidence->getHeaders(); foreach ($headers as $key => $value) { if (is_array($value) || is_object($value)) { foreach ($value as $index => $subvalue) { $report['headers']["{$key}{$index}"] = "{$subvalue}"; } } else { $report['headers']["{$key}"] = $value; } } /* * Sometimes Spamcop has a trouble adding the correct fields. The IP is pretty * normal to add. In a last attempt we will try to fetch the IP from the body ourselves */ if (empty($report['Source-IP'])) { preg_match("/Email from (?<ip>[a-f0-9:\\.]+) \\/ " . preg_quote($report['Received-Date']) . "/s", $this->arfMail['message'], $regs); if (!empty($regs['ip']) && !filter_var($regs['ip'], FILTER_VALIDATE_IP) === false) { $report['Source-IP'] = $regs['ip']; } preg_match("/from: (?<ip>[a-f0-9:\\.]+)\r?\n?\r\n/s", $this->parsedMail->getMessageBody(), $regs); if (!empty($regs['ip']) && !filter_var($regs['ip'], FILTER_VALIDATE_IP) === false) { $report['Source-IP'] = $regs['ip']; } } $reports[] = $report; return $reports; }