Fully Tested Mailparse Extension Wrapper for PHP 5.4+
Ejemplo n.º 1
0
 /**
  * Parse attachments
  * @return array    Returns array with failed or success data
  *                  (See parser-common/src/Parser.php) for more info.
  */
 public function parse()
 {
     if ($this->arfMail !== true) {
         $this->feedName = 'default';
         // As this is a generic FBL parser we need to see which was the source and add the name
         // to the report, so its origin is clearly shown.
         $source = $this->parsedMail->getHeader('from');
         foreach (config("{$this->configBase}.parser.aliases") as $from => $alias) {
             if (preg_match($from, $source)) {
                 // If there is an alias, prefer that name instead of the from address
                 $source = $alias;
                 // If there is an more specific feed configuration prefer that config over the default
                 if (!empty(config("{$this->configBase}.feeds.{$source}"))) {
                     $this->feedName = $source;
                 }
             }
         }
         // If feed is known and enabled, validate data and save report
         if ($this->isKnownFeed() && $this->isEnabledFeed()) {
             // To get some more consitency, remove "\r" from the report.
             $this->arfMail['report'] = str_replace("\r", "", $this->arfMail['report']);
             // Build up the report
             preg_match_all("/([\\w\\-]+): (.*)[ ]*\n/m", $this->arfMail['report'], $matches);
             $report = array_combine($matches[1], $matches[2]);
             if (empty($report['Received-Date'])) {
                 if (!empty($report['Arrival-Date'])) {
                     $report['Received-Date'] = $report['Arrival-Date'];
                     unset($report['Arrival-Date']);
                 }
             }
             // Now parse the headers from the spam messages and add it to the report
             if (!empty($this->arfMail['evidence'])) {
                 $spamMessage = new MimeParser();
                 $spamMessage->setText($this->arfMail['evidence']);
                 $report['headers'] = $spamMessage->getHeaders();
             } else {
                 $this->failed('The e-mail received at the parser is not RFC822 compliant, and therefor not a FBL message');
             }
             // Also add the spam message body to the report
             $report['body'] = $spamMessage->getMessageBody();
             // Sanity check
             if ($this->hasRequiredFields($report) === true) {
                 // incident has all requirements met, filter and add!
                 $report = $this->applyFilters($report);
                 $incident = new Incident();
                 $incident->source = $source;
                 // FeedName
                 $incident->source_id = false;
                 $incident->ip = $report['Source-IP'];
                 $incident->domain = false;
                 $incident->class = config("{$this->configBase}.feeds.{$this->feedName}.class");
                 $incident->type = config("{$this->configBase}.feeds.{$this->feedName}.type");
                 $incident->timestamp = strtotime($report['Received-Date']);
                 $incident->information = json_encode($report);
                 $this->incidents[] = $incident;
             }
         }
     }
     return $this->success();
 }
Ejemplo n.º 2
0
 /**
  * Create and return a Parser class and it's configuration
  * @param  \PhpMimeMailParser\Parser $parsedMail
  * @param  array $arfMail
  * @return object parser
  */
 public static function create($parsedMail, $arfMail)
 {
     /**
      * Loop through the parser list and try to find a match by
      * validating the send or the body according to the parsers'
      * configuration.
      */
     $parsers = Factory::getParsers();
     foreach ($parsers as $parserName) {
         $parserClass = 'AbuseIO\\Parsers\\' . $parserName;
         // Parser is enabled, see if we can match it's sender_map or body_map
         if (config("parsers.{$parserName}.parser.enabled") === true) {
             // Check validity of the 'report_file' setting before we continue
             // If no report_file is used, continue w/o validation
             $report_file = config("parsers.{$parserName}.parser.report_file");
             if ($report_file == null || is_string($report_file) && isValidRegex($report_file)) {
                 $isValidReport = true;
             } else {
                 $isValidReport = false;
                 Log::warning('AbuseIO\\Parsers\\Factory: ' . "The parser {$parserName} has an invalid value for 'report_file' (not a regex).");
                 break;
             }
             // Check the sender address
             foreach (config("parsers.{$parserName}.parser.sender_map") as $senderMap) {
                 if (isValidRegex($senderMap)) {
                     if (preg_match($senderMap, $parsedMail->getHeader('from')) && $isValidReport) {
                         return new $parserClass($parsedMail, $arfMail);
                     }
                 } else {
                     Log::warning('AbuseIO\\Parsers\\Factory: ' . "The parser {$parserName} has an invalid value for 'sender_map' (not a regex).");
                 }
             }
             // If no valid sender is found, check the body
             foreach (config("parsers.{$parserName}.parser.body_map") as $bodyMap) {
                 if (isValidRegex($bodyMap)) {
                     if (preg_match($bodyMap, $parsedMail->getMessageBody()) && $isValidReport) {
                         return new $parserClass($parsedMail, $arfMail);
                     }
                     if ($arfMail !== false) {
                         foreach ($arfMail as $mailPart) {
                             if (preg_match($bodyMap, $mailPart)) {
                                 return new $parserClass($parsedMail, $arfMail);
                             }
                         }
                     }
                 } else {
                     Log::warning('AbuseIO\\Parsers\\Factory: ' . "The parser {$parserName} has an invalid value for 'body_map' (not a regex).");
                 }
             }
         } else {
             Log::info('AbuseIO\\Parsers\\Factory: ' . "The parser {$parserName} has been disabled and will not be used for this message.");
         }
     }
     // No valid parsers found
     return false;
 }
Ejemplo n.º 3
0
 /**
  * Parse the given email resource
  *
  * @param  string $file path/to/file
  * @return Parser
  */
 public function parseFile($file)
 {
     if (!file_exists($file)) {
         throw new Exception("File {$file} does not exist.");
     }
     $this->lines = file($file);
     $this->parser->setPath($file);
     $bounceReason = $this->findBounceReason();
     fputcsv($this->csv, array($this->findRecipient(), key($bounceReason), current($bounceReason)), $this->csvDelimiter, $this->csvEnclosure);
     return $this;
 }
Ejemplo n.º 4
0
 public function create(string $mailbox, Horde_Imap_Client_Data_Fetch $hordeEmail) : Email
 {
     // Parse the message body
     $parser = new Parser();
     $parser->setText($hordeEmail->getFullMsg());
     $htmlContent = utf8_encode((string) $parser->getMessageBody('html'));
     $textContent = utf8_encode((string) $parser->getMessageBody('text'));
     // Filter HTML body to have only safe HTML
     $htmlContent = trim($this->htmlFilter->purify($htmlContent));
     // If no HTML content, use the text content
     if ($htmlContent == '') {
         $htmlContent = nl2br($textContent);
     }
     // The envelope contains the headers
     $envelope = $hordeEmail->getEnvelope();
     $from = [];
     foreach ($envelope->from as $hordeFrom) {
         /** @var Horde_Mail_Rfc822_Address $hordeFrom */
         if ($hordeFrom->bare_address) {
             $from[] = new EmailAddress($hordeFrom->bare_address, $hordeFrom->personal);
         }
     }
     $to = [];
     foreach ($envelope->to as $hordeTo) {
         /** @var Horde_Mail_Rfc822_Address $hordeTo */
         if ($hordeTo->bare_address) {
             $to[] = new EmailAddress($hordeTo->bare_address, $hordeTo->personal);
         }
     }
     $messageId = $this->parseMessageId($envelope->message_id);
     $inReplyTo = $this->parseMessageId($envelope->in_reply_to);
     $message = new Email((string) $hordeEmail->getUid(), $messageId, $mailbox, $envelope->subject, $htmlContent, $textContent, $from, $to, $inReplyTo);
     $date = new DateTime();
     $date->setTimestamp($envelope->date->getTimestamp());
     $message->setDate($date);
     $flags = $hordeEmail->getFlags();
     if (in_array(Horde_Imap_Client::FLAG_SEEN, $flags)) {
         $message->setRead(true);
     }
     return $message;
 }
 /**
  * @param $rawMessage
  *
  * @return \MailChecker\Models\Message
  */
 protected function getMessage($rawMessage)
 {
     $parser = new MailParser();
     $parser->setText($rawMessage);
     $headers = array_change_key_case($parser->getHeaders(), CASE_LOWER);
     $message = new Message();
     try {
         $message->setDate(new \DateTime($headers['date']));
     } catch (\Exception $e) {
         // Can't recognize date time format
         // TODO add config option for date time format parsing
         $message->setDate(new \DateTime());
     }
     $message->setSubject($headers['subject']);
     $message->setFrom($headers['from']);
     $message->setTo($headers['to']);
     if (isset($headers['cc'])) {
         $message->setCc($headers['cc']);
     }
     foreach ($parser->getParts() as $part) {
         if (in_array($part['content-type'], ['text/plain', 'text/html'])) {
             $body = new Body();
             $body->setContentType($part['content-type']);
             $body->setCharset(isset($part['content-charset']) ? $part['content-charset'] : null);
             $body->setEncoding(isset($part['transfer-encoding']) ? $part['transfer-encoding'] : null);
             $start = $part['starting-pos-body'];
             $end = $part['ending-pos-body'];
             $body->setBody(substr($rawMessage, $start, $end - $start));
             $message->addBody($body);
         }
     }
     /** @var \PhpMimeMailParser\Attachment $messageAttachment */
     foreach ($parser->getAttachments() as $messageAttachment) {
         $attachment = new Attachment();
         $attachment->setType($messageAttachment->getContentType());
         $attachment->setFilename($messageAttachment->getFilename());
         $message->addAttachment($attachment);
     }
     return $message;
 }
Ejemplo n.º 6
0
 /**
  * @dataProvider provideData
  */
 public function testFromStream($mid, $subjectExpected, $fromExpected, $toExpected, $textExpected, $htmlExpected, $attachmentsExpected, $countEmbeddedExpected)
 {
     //Init
     $file = __DIR__ . '/mails/' . $mid;
     $attach_dir = __DIR__ . '/mails/attach_' . $mid . '/';
     //Load From Path
     $Parser = new Parser();
     $Parser->setStream(fopen($file, 'r'));
     //Test Header : subject
     $this->assertEquals($subjectExpected, $Parser->getHeader('subject'));
     $this->assertArrayHasKey('subject', $Parser->getHeaders());
     //Test Header : from
     $this->assertEquals($fromExpected, $Parser->getHeader('from'));
     $this->assertArrayHasKey('from', $Parser->getHeaders());
     //Test Header : to
     $this->assertEquals($toExpected, $Parser->getHeader('to'));
     $this->assertArrayHasKey('to', $Parser->getHeaders());
     //Test Invalid Header
     $this->assertFalse($Parser->getHeader('azerty'));
     $this->assertArrayNotHasKey('azerty', $Parser->getHeaders());
     //Test  Body : text
     if ($textExpected[0] == 'COUNT') {
         $this->assertEquals($textExpected[1], substr_count($Parser->getMessageBody('text'), $textExpected[2]));
     } elseif ($textExpected[0] == 'MATCH') {
         $this->assertEquals($textExpected[1], $Parser->getMessageBody('text'));
     }
     //Test Body : html
     if ($htmlExpected[0] == 'COUNT') {
         $this->assertEquals($htmlExpected[1], substr_count($Parser->getMessageBody('html'), $htmlExpected[2]));
     } elseif ($htmlExpected[0] == 'MATCH') {
         $this->assertEquals($htmlExpected[1], $Parser->getMessageBody('html'));
     }
     //Test Nb Attachments
     $attachments = $Parser->getAttachments();
     $this->assertEquals(count($attachmentsExpected), count($attachments));
     $iterAttachments = 0;
     //Test Attachments
     $attachmentsEmbeddedToCheck = array();
     if (count($attachmentsExpected) > 0) {
         //Save attachments
         $Parser->saveAttachments($attach_dir);
         foreach ($attachmentsExpected as $attachmentExpected) {
             //Test Exist Attachment
             $this->assertTrue(file_exists($attach_dir . $attachmentExpected[0]));
             //Test Filename Attachment
             $this->assertEquals($attachmentExpected[0], $attachments[$iterAttachments]->getFilename());
             //Test Size Attachment
             $this->assertEquals($attachmentExpected[1], filesize($attach_dir . $attachments[$iterAttachments]->getFilename()));
             //Test Inside Attachment
             if ($attachmentExpected[2] != '' && $attachmentExpected[3] > 0) {
                 $fileContent = file_get_contents($attach_dir . $attachments[$iterAttachments]->getFilename(), FILE_USE_INCLUDE_PATH);
                 $this->assertEquals($attachmentExpected[3], substr_count($fileContent, $attachmentExpected[2]));
                 $this->assertEquals($attachmentExpected[3], substr_count($attachments[$iterAttachments]->getContent(), $attachmentExpected[2]));
             }
             //Test ContentType Attachment
             $this->assertEquals($attachmentExpected[4], $attachments[$iterAttachments]->getContentType());
             //Test ContentDisposition Attachment
             $this->assertEquals($attachmentExpected[5], $attachments[$iterAttachments]->getContentDisposition());
             //Test md5 of Headers Attachment
             $this->assertEquals($attachmentExpected[6], md5(serialize($attachments[$iterAttachments]->getHeaders())));
             //Save embedded Attachments to check
             if ($attachmentExpected[7] != '') {
                 array_push($attachmentsEmbeddedToCheck, $attachmentExpected[7]);
             }
             //Remove Attachment
             unlink($attach_dir . $attachments[$iterAttachments]->getFilename());
             $iterAttachments++;
         }
         //Remove Attachment Directory
         rmdir($attach_dir);
     } else {
         $this->assertFalse($Parser->saveAttachments($attach_dir));
     }
     //Test embedded Attachments
     $htmlEmbedded = $Parser->getMessageBody('htmlEmbedded');
     $this->assertEquals($countEmbeddedExpected, substr_count($htmlEmbedded, "data:"));
     if (!empty($attachmentsEmbeddedToCheck)) {
         foreach ($attachmentsEmbeddedToCheck as $itemExpected) {
             $this->assertEquals(1, substr_count($htmlEmbedded, $itemExpected));
         }
     }
 }
Ejemplo n.º 7
0
 /**
  * @param $filename
  *
  * @return bool
  */
 public function getAttachment($filename)
 {
     $data = Storage::get($this->filename);
     $email = new MimeParser();
     $email->setText($data);
     foreach ($email->getAttachments() as $attachment) {
         if ($attachment->getFilename() == $filename) {
             return $attachment;
         }
     }
     return false;
 }
Ejemplo n.º 8
0
 /**
  * Shows the evidence for this ticket as webpage.
  *
  * @param Ticket $ticket        Ticket Model
  * @param integer $evidenceId
  * @return \Illuminate\Http\Response
  */
 public function viewEvidence(Ticket $ticket, $evidenceId)
 {
     $evidence = Evidence::find($evidenceId);
     if (!$evidence) {
         return Redirect::route('admin.tickets.show', $ticket->id)->with('message', 'The evidence is no longer available for this event.');
     }
     if (is_file($evidence->filename)) {
         $evidenceData = file_get_contents($evidence->filename);
         $evidenceParsed = new MimeParser();
         $evidenceParsed->setText($evidenceData);
         $filesystem = new Filesystem();
         $evidenceTempDir = "/tmp/abuseio/cache/{$ticket->id}/{$evidenceId}/";
         if (!$filesystem->isDirectory($evidenceTempDir)) {
             if (!$filesystem->makeDirectory($evidenceTempDir, 0755, true)) {
                 Log::error(get_class($this) . ': ' . 'Unable to create temp directory: ' . $evidenceTempDir);
             }
             $evidenceParsed->saveAttachments($evidenceTempDir);
         }
         return view('tickets.evidence')->with('ticket', $ticket)->with('evidence', $evidenceParsed)->with('evidenceId', $evidenceId)->with('evidenceTempDir', $evidenceTempDir)->with('auth_user', $this->auth_user);
     } else {
         return Redirect::route('admin.tickets.show', $ticket->id)->with('message', 'ERROR: The file was not available on the filesystem.');
     }
 }
Ejemplo n.º 9
0
 /**
  * Execute the command
  *
  * @return void
  */
 public function handle()
 {
     Log::info('(JOB ' . getmypid() . ') ' . get_class($this) . ': ' . 'Queued worker is starting the processing of email file: ' . $this->filename);
     $filesystem = new Filesystem();
     $rawEmail = $filesystem->get($this->filename);
     $parsedMail = new MimeParser();
     $parsedMail->setText($rawEmail);
     // Sanity checks
     if (empty($parsedMail->getHeader('from')) || empty($parsedMail->getMessageBody())) {
         Log::warning('(JOB ' . getmypid() . ') ' . get_class($this) . ': ' . 'Missing e-mail headers from and/or empty body: ' . $this->filename);
         $this->alertAdmin();
         return;
     }
     // Ignore email from our own notification address to prevent mail loops
     if (preg_match('/' . Config::get('main.notifications.from_address') . '/', $parsedMail->getHeader('from'))) {
         Log::warning('(JOB ' . getmypid() . ') ' . get_class($this) . ': ' . 'Loop prevention: Ignoring email from self ' . Config::get('main.notifications.from_address'));
         $this->alertAdmin();
         return;
     }
     // Start with detecting valid ARF e-mail
     $attachments = $parsedMail->getAttachments();
     $arfMail = [];
     foreach ($attachments as $attachment) {
         if ($attachment->contentType == 'message/feedback-report') {
             $arfMail['report'] = $attachment->getContent();
         }
         if ($attachment->contentType == 'message/rfc822') {
             $arfMail['evidence'] = utf8_encode($attachment->getContent());
         }
         if ($attachment->contentType == 'text/plain') {
             $arfMail['message'] = $attachment->getContent();
         }
     }
     /*
      * Sometimes the mime header does not set the main message correctly. This is ment as a fallback and will
      * use the original content body (which is basicly the same mime element). But only fallback if we actually
      * have a RFC822 message with a feedback report.
      */
     if (empty($arfMail['message']) && isset($arfMail['report']) && isset($arfMail['evidence'])) {
         $arfMail['message'] = $parsedMail->getMessageBody();
     }
     // If we do not have a complete e-mail, then we empty the perhaps partially filled arfMail
     // which is useless, hence reset to false
     if (!isset($arfMail['report']) || !isset($arfMail['evidence']) || !isset($arfMail['message'])) {
         $arfMail = false;
     }
     // Asking ParserFactory for an object based on mappings, or die trying
     $parser = ParserFactory::create($parsedMail, $arfMail);
     if ($parser !== false) {
         $parserResult = $parser->parse();
     } else {
         Log::error('(JOB ' . getmypid() . ') ' . get_class($this) . ': ' . ': No parser available to handle message from : ' . $parsedMail->getHeader('from') . ' with subject: ' . $parsedMail->getHeader('subject'));
         $this->alertAdmin();
         return;
     }
     if ($parserResult !== false && $parserResult['errorStatus'] === true) {
         Log::error('(JOB ' . getmypid() . ') ' . get_class($parser) . ': ' . ': Parser has ended with fatal errors ! : ' . $parserResult['errorMessage']);
         $this->alertAdmin();
         return;
     } else {
         Log::info('(JOB ' . getmypid() . ') ' . get_class($parser) . ': ' . ': Parser completed with ' . $parserResult['warningCount'] . ' warnings and collected ' . count($parserResult['data']) . ' events to save');
     }
     if ($parserResult['warningCount'] !== 0 && Config::get('main.emailparser.notify_on_warnings') === true) {
         Log::error('(JOB ' . getmypid() . ') ' . get_class($this) . ': ' . 'Configuration has warnings set as critical and ' . $parserResult['warningCount'] . ' warnings were detected. Sending alert to administrator');
         $this->alertAdmin();
         return;
     }
     if (count($parserResult['data']) !== 0) {
         // Call validator
         $validator = new EventsValidate();
         $validatorResult = $validator->check($parserResult['data']);
         if ($validatorResult['errorStatus'] === true) {
             Log::error('(JOB ' . getmypid() . ') ' . get_class($validator) . ': ' . 'Validator has ended with errors ! : ' . $validatorResult['errorMessage']);
             $this->alertAdmin();
             return;
         } else {
             Log::info('(JOB ' . getmypid() . ') ' . get_class($validator) . ': ' . 'Validator has ended without errors');
         }
         /**
          * save evidence into table
          **/
         $evidence = new Evidence();
         $evidence->filename = $this->filename;
         $evidence->sender = $parsedMail->getHeader('from');
         $evidence->subject = $parsedMail->getHeader('subject');
         $evidence->save();
         /**
          * call saver
          **/
         $saver = new EventsSave();
         $saverResult = $saver->save($parserResult['data'], $evidence->id);
         /**
          * We've hit a snag, so we are gracefully killing ourselves
          * after we contact the admin about it. EventsSave should never
          * end with problems unless the mysql died while doing transactions
          **/
         if ($saverResult['errorStatus'] === true) {
             Log::error('(JOB ' . getmypid() . ') ' . get_class($saver) . ': ' . 'Saver has ended with errors ! : ' . $saverResult['errorMessage']);
             $this->alertAdmin();
             return;
         } else {
             Log::info('(JOB ' . getmypid() . ') ' . get_class($saver) . ': ' . 'Saver has ended without errors');
         }
     } else {
         Log::warning('(JOB ' . getmypid() . ') ' . get_class($this) . ': ' . 'Parser did not return any events therefore skipping validation and saving a empty event set');
     }
     Log::info('(JOB ' . getmypid() . ') ' . get_class($this) . ': ' . 'Queued worker has ended the processing of email file: ' . $this->filename);
 }
Ejemplo n.º 10
0
 /**
  * Execute the command.
  *
  * @return void
  */
 public function handle()
 {
     Log::info(get_class($this) . ': Queued worker is starting the processing of email file: ' . $this->filename);
     $filesystem = new Filesystem();
     $rawEmail = $filesystem->get($this->filename);
     $parsedMail = new MimeParser();
     $parsedMail->setText($rawEmail);
     // Sanity checks
     if (empty($parsedMail->getHeader('from')) || empty($parsedMail->getMessageBody())) {
         Log::warning(get_class($this) . 'Validation failed on: ' . $this->filename);
         $this->exception();
     }
     // Ignore email from our own notification address to prevent mail loops
     if (preg_match('/' . Config::get('main.notifications.from_address') . '/', $parsedMail->getHeader('from'))) {
         Log::warning(get_class($this) . 'Loop prevention: Ignoring email from self ' . Config::get('main.notifications.from_address'));
         $this->exception();
     }
     // Start with detecting valid ARF e-mail
     $attachments = $parsedMail->getAttachments();
     $arfMail = [];
     foreach ($attachments as $attachment) {
         if ($attachment->contentType == 'message/feedback-report') {
             $arfMail['report'] = $attachment->getContent();
         }
         if ($attachment->contentType == 'message/rfc822') {
             $arfMail['evidence'] = $attachment->getContent();
         }
         if ($attachment->contentType == 'text/plain') {
             $arfMail['message'] = $attachment->getContent();
         }
     }
     // If we do not have a complete e-mail, then we empty the perhaps partially filled arfMail
     // which is useless, hence reset to false
     if (!isset($arfMail['report']) || !isset($arfMail['evidence']) || !isset($arfMail['message'])) {
         $arfMail = false;
     }
     // Asking GetParser for an object based on mappings, or die trying
     $parser = GetParser::object($parsedMail, $arfMail);
     $result = false;
     $events = false;
     if ($parser !== false) {
         $result = $parser->parse();
     } else {
         Log::error(get_class($this) . ': Unable to handle message from: ' . $parsedMail->getHeader('from') . ' with subject: ' . $parsedMail->getHeader('subject'));
         $this->exception();
     }
     if ($result !== false && $result['errorStatus'] !== true) {
         Log::info(get_class($parser) . ': Parser as ended without errors. Collected ' . count($result['data']) . ' events to save');
         $events = $result['data'];
     } else {
         Log::error(get_class($parser) . ': Parser as ended with errors ! : ' . $result['errorMessage']);
         $this->exception();
     }
     // call validater
     $validator = new EventsValidate($events);
     $return = $validator->handle();
     if ($return['errorStatus'] === false) {
         Log::error(get_class($validator) . ': Validator as ended with errors ! : ' . $result['errorMessage']);
         $this->exception();
     } else {
         Log::info(get_class($validator) . ': Validator as ended without errors');
     }
     // save evidence into table
     $evidence = new Evidence();
     $evidence->filename = $this->filename;
     $evidence->sender = $parsedMail->getHeader('from');
     $evidence->subject = $parsedMail->getHeader('subject');
     $evidence->save();
     // call saver
     $saver = new EventsSave($events, $evidence->id);
     $return = $saver->handle();
     if ($return['errorStatus'] === false) {
         Log::error(get_class($saver) . ': Saver as ended with errors ! : ' . $result['errorMessage']);
         $this->exception();
     } else {
         Log::info(get_class($saver) . ': Saver as ended without errors');
     }
 }
Ejemplo n.º 11
0
 /**
  * Execute the console command.
  *
  * @return bool
  */
 public function handle()
 {
     $account = Account::getSystemAccount();
     if (empty($this->option('start')) && empty($this->option('prepare')) && empty($this->option('clean'))) {
         $this->error('You need to either prepare or start the migration. try --help');
         die;
     }
     if (!empty($this->option('clean'))) {
         $this->warn('This will remove all data from the database except prepared data. Do not run ' . 'this in producion and only when a migration has gone bad and you want to restart it');
         if ($this->confirm('Do you wish to continue? [y|N]')) {
             $this->info('starting clean up');
             DB::statement('SET foreign_key_checks=0');
             Note::truncate();
             Event::truncate();
             Ticket::truncate();
             Netblock::truncate();
             Contact::truncate();
             DB::statement('SET foreign_key_checks=1');
         } else {
             $this->info('cancelled clean up');
         }
     }
     /*
      * Combine the use of start/stop and threading here for usage with prepare and start of migration
      */
     $startFrom = $this->option('startfrom');
     $endWith = $this->option('endwith');
     if (!empty($this->option('prepare')) || !empty($this->option('start'))) {
         if (!empty($this->option('threaded'))) {
             if (empty($this->option('threadid')) || empty($this->option('threadsize'))) {
                 $this->error('threadid and threadsize are required to calculate this threads start/stop ID');
                 die;
             }
             $this->info("*** using threaded mode, instance ID {$this->option('threadid')}");
             $startFrom = $this->option('threadid') * $this->option('threadsize') - $this->option('threadsize') + 1;
             $endWith = $startFrom + $this->option('threadsize') - 1;
             $this->info("*** starting with ticket {$startFrom} " . "and ending with ticket {$endWith} ");
         }
     }
     if (!empty($this->option('prepare'))) {
         $this->info('building required evidence cache files');
         $filesystem = new Filesystem();
         $path = storage_path() . '/migration/';
         umask(07);
         if (!$filesystem->isDirectory($path)) {
             // If a datefolder does not exist, then create it or die trying
             if (!$filesystem->makeDirectory($path, 0770)) {
                 $this->error('Unable to create directory: ' . $path);
                 $this->exception();
             }
             if (!is_dir($path)) {
                 $this->error('Path vanished after write: ' . $path);
                 $this->exception();
             }
             chgrp($path, config('app.group'));
         }
         DB::setDefaultConnection('abuseio3');
         $evidenceRows = DB::table('Evidence')->where('id', '>=', $startFrom)->where('id', '<=', $endWith);
         $migrateCount = $evidenceRows->count();
         $evidences = $evidenceRows->get();
         $this->output->progressStart($migrateCount);
         // If there are now rows to do, advance to complete
         if ($migrateCount === 0) {
             $this->output->progressAdvance();
             echo ' nothing to do, because there are no records in this selection';
         }
         foreach ($evidences as $evidence) {
             $this->output->progressAdvance();
             // Before we go into an error, lets see if this evidence was even linked to any ticket at all
             // If not we can ignore the error and just smile and wave
             $evidenceLinks = DB::table('EvidenceLinks')->where('EvidenceID', '=', $evidence->ID)->get();
             if (count($evidenceLinks) === 0) {
                 echo " skipping unlinked evidence ID {$evidence->ID}         ";
                 continue;
             }
             $filename = $path . "evidence_id_{$evidence->ID}.data";
             if (is_file($filename)) {
                 // Check weither the evidence was actually created in the database
                 DB::setDefaultConnection('mysql');
                 $evidenceCheck = Evidence::where('id', $evidence->ID);
                 if ($evidenceCheck->count() !== 1) {
                     $this->error("file {$filename} exists however not in database. try deleting it to retry");
                     die;
                 }
                 DB::setDefaultConnection('abuseio3');
                 continue;
             } else {
                 echo " working on evidence ID {$evidence->ID}                    ";
             }
             $rawEmail = $evidence->Data;
             $parsedMail = new MimeParser();
             $parsedMail->setText($rawEmail);
             // Start with detecting valid ARF e-mail
             $attachments = $parsedMail->getAttachments();
             $arfMail = [];
             foreach ($attachments as $attachment) {
                 if ($attachment->contentType == 'message/feedback-report') {
                     $arfMail['report'] = $attachment->getContent();
                 }
                 if ($attachment->contentType == 'message/rfc822') {
                     $arfMail['evidence'] = utf8_encode($attachment->getContent());
                 }
                 if ($attachment->contentType == 'text/plain') {
                     $arfMail['message'] = $attachment->getContent();
                 }
             }
             if (empty($arfMail['message']) && isset($arfMail['report']) && isset($arfMail['evidence'])) {
                 $arfMail['message'] = $parsedMail->getMessageBody();
             }
             // If we do not have a complete e-mail, then we empty the perhaps partially filled arfMail
             // which is useless, hence reset to false
             if (!isset($arfMail['report']) || !isset($arfMail['evidence']) || !isset($arfMail['message'])) {
                 $arfMail = false;
             }
             // Asking ParserFactory for an object based on mappings, or die trying
             $parser = ParserFactory::create($parsedMail, $arfMail);
             if ($parser !== false) {
                 $parserResult = $parser->parse();
             } else {
                 $this->error('No parser available to handle message ' . $evidence->ID . ' from : ' . $evidence->Sender . ' with subject: ' . $evidence->Subject);
                 continue;
             }
             if ($parserResult !== false && $parserResult['errorStatus'] === true) {
                 $this->error('Parser has ended with fatal errors ! : ' . $parserResult['errorMessage']);
                 $this->exception();
             }
             if ($parserResult['warningCount'] !== 0) {
                 $this->error('Configuration has warnings set as critical and ' . $parserResult['warningCount'] . ' warnings were detected.');
                 //var_dump($rawEmail);
                 $this->exception();
             }
             // Write the evidence into the archive
             $evidenceWrite = new EvidenceSave();
             $evidenceData = $rawEmail;
             $evidenceFile = $evidenceWrite->save($evidenceData);
             // Save the file reference into the database
             $evidenceSave = new Evidence();
             $evidenceSave->id = $evidence->ID;
             $evidenceSave->filename = $evidenceFile;
             $evidenceSave->sender = $parsedMail->getHeader('from');
             $evidenceSave->subject = $parsedMail->getHeader('subject');
             $incidentsProcess = new IncidentsProcess($parserResult['data'], $evidenceSave);
             /*
              * Because google finds it 'obvious' not to include the IP address relating to abuse
              * the IP field might now be empty with reparsing if the domain/label does not resolve
              * anymore. For these cases we need to lookup the ticket that was linked to the evidence
              * match the domain and retrieve its IP.
              */
             foreach ($parserResult['data'] as $index => $incident) {
                 if ($incident->source == 'Google Safe Browsing' && $incident->domain != false && $incident->ip == '127.0.0.1') {
                     // Get the list of tickets related to this evidence
                     $evidenceLinks = DB::table('EvidenceLinks')->where('EvidenceID', '=', $evidence->ID)->get();
                     // For each ticket check if the domain name is matching the evidence we need to update
                     foreach ($evidenceLinks as $evidenceLink) {
                         $ticket = DB::table('Reports')->where('ID', '=', $evidenceLink->ReportID)->first();
                         if ($ticket->Domain == $incident->domain) {
                             $incident->ip = $ticket->IP;
                         }
                     }
                     // Update the original object by overwriting it
                     $parserResult['data'][$index] = $incident;
                 }
             }
             // Only continue if not empty, empty set is acceptable (exit OK)
             if (!$incidentsProcess->notEmpty()) {
                 $this->warn("No evidence build, no results from parser for {$evidence->ID}");
                 continue;
             }
             // Validate the data set
             if (!$incidentsProcess->validate()) {
                 $this->error('Validation failed of object.');
                 $this->exception();
             }
             $incidents = [];
             foreach ($parserResult['data'] as $incident) {
                 $incidents[$incident->ip][] = $incident;
             }
             DB::setDefaultConnection('mysql');
             $evidenceSave->save();
             DB::setDefaultConnection('abuseio3');
             $output = ['evidenceId' => $evidence->ID, 'evidenceData' => $evidence->Data, 'incidents' => $incidents, 'newId' => $evidenceSave->id];
             if ($filesystem->put($filename, json_encode($output)) === false) {
                 $this->error('Unable to write file: ' . $filename);
                 return false;
             }
         }
         $this->output->progressFinish();
     }
     if (!empty($this->option('start'))) {
         if (empty($this->option('skipcontacts'))) {
             $this->info('starting migration - phase 1 - contact data');
             DB::setDefaultConnection('abuseio3');
             $customers = DB::table('Customers')->get();
             DB::setDefaultConnection('mysql');
             $this->output->progressStart(count($customers));
             foreach ($customers as $customer) {
                 $newContact = new Contact();
                 $newContact->reference = $customer->Code;
                 $newContact->name = $customer->Name;
                 $newContact->email = $customer->Contact;
                 $newContact->auto_notify = $customer->AutoNotify;
                 $newContact->enabled = 1;
                 $newContact->account_id = $account->id;
                 $newContact->created_at = Carbon::parse($customer->LastModified);
                 $newContact->updated_at = Carbon::parse($customer->LastModified);
                 $validation = Validator::make($newContact->toArray(), Contact::createRules());
                 if ($validation->fails()) {
                     $message = implode(' ', $validation->messages()->all());
                     $this->error('fatal error while creating contacts :' . $message);
                     $this->exception();
                 } else {
                     $newContact->save();
                 }
                 $this->output->progressAdvance();
                 echo " Working on contact {$customer->Code}      ";
             }
             $this->output->progressFinish();
         } else {
             $this->info('skipping migration - phase 1 - contact data');
         }
         if (empty($this->option('skipnetblocks'))) {
             $this->info('starting migration - phase 2 - netblock data');
             DB::setDefaultConnection('abuseio3');
             $netblocks = DB::table('Netblocks')->get();
             DB::setDefaultConnection('mysql');
             $this->output->progressStart(count($netblocks));
             foreach ($netblocks as $netblock) {
                 $contact = FindContact::byId($netblock->CustomerCode);
                 if ($contact->reference != $netblock->CustomerCode) {
                     $this->error('Contact lookup failed, mismatched results');
                     $this->{$this}->exception();
                 }
                 $newNetblock = new Netblock();
                 $newNetblock->first_ip = long2ip($netblock->begin_in);
                 $newNetblock->last_ip = long2ip($netblock->end_in);
                 $newNetblock->description = 'Imported from previous AbuseIO version which did not include a description';
                 $newNetblock->contact_id = $contact->id;
                 $newNetblock->enabled = 1;
                 $newNetblock->created_at = Carbon::parse($netblock->LastModified);
                 $newNetblock->updated_at = Carbon::parse($netblock->LastModified);
                 $validation = Validator::make($newNetblock->toArray(), Netblock::createRules($newNetblock));
                 if ($validation->fails()) {
                     $message = implode(' ', $validation->messages()->all());
                     $this->error('fatal error while creating contacts :' . $message);
                     $this->exception();
                 } else {
                     $newNetblock->save();
                 }
                 $this->output->progressAdvance();
                 echo ' Working on netblock ' . long2ip($netblock->begin_in) . '       ';
             }
             $this->output->progressFinish();
         } else {
             $this->info('skipping migration - phase 2 - netblock data');
         }
         if (empty($this->option('skiptickets'))) {
             $this->info('starting migration - phase 3 - ticket and evidence data');
             DB::setDefaultConnection('abuseio3');
             $ticketRows = DB::table('Reports')->where('id', '>=', $startFrom)->where('id', '<=', $endWith);
             $migrateCount = $ticketRows->count();
             $tickets = $ticketRows->get();
             DB::setDefaultConnection('mysql');
             $this->output->progressStart($migrateCount);
             // If there are now rows to do, advance to complete
             if ($migrateCount === 0) {
                 $this->output->progressAdvance();
                 echo ' nothing to do, because there are no records in this selection';
             }
             foreach ($tickets as $ticket) {
                 // Get the list of evidence ID's related to this ticket
                 DB::setDefaultConnection('abuseio3');
                 $evidenceLinks = DB::table('EvidenceLinks')->where('ReportID', '=', $ticket->ID)->get();
                 DB::setDefaultConnection('mysql');
                 // DO NOT REMOVE! Legacy versions (1.0 / 2.0) have imports without evidence.
                 // These dont have any linked evidence and will require a manual building of evidence
                 // for now we ignore them. This will not affect any 3.x installations
                 if ($ticket->CustomerName == 'Imported from AbuseReporter' || !empty(json_decode($ticket->Information)->importnote)) {
                     // Manually build the evidence
                     $this->output->progressAdvance();
                     echo " Working on events from ticket {$ticket->ID}";
                     $this->replayTicket($ticket, $account);
                     continue;
                 }
                 if (count($evidenceLinks) != (int) $ticket->ReportCount) {
                     // Count does not match, known 3.0 limitation related to not always saving all the data
                     // so we will do a little magic to fix that
                     $this->output->progressAdvance();
                     echo " Working on events from ticket {$ticket->ID}";
                     $this->replayTicket($ticket, $account);
                     continue;
                 } else {
                     // Just work as normal
                     $this->output->progressAdvance();
                     echo " Working on events from ticket {$ticket->ID}";
                     $newTicket = $this->createTicket($ticket, $account);
                     // Create all the events
                     foreach ($evidenceLinks as $evidenceLink) {
                         $path = storage_path() . '/migration/';
                         $filename = $path . "evidence_id_{$evidenceLink->EvidenceID}.data";
                         if (!is_file($filename)) {
                             $this->error('missing cache file ');
                             $this->exception();
                         }
                         $evidence = json_decode(file_get_contents($filename));
                         $evidenceID = (int) $evidence->evidenceId;
                         $incidents = $evidence->incidents;
                         // Yes we only grab nr 0 from the array, because that is what the old aggregator did
                         // which basicly ignored a few incidents because they werent considered unique (which
                         // they were with the domain name)
                         $ip = $newTicket->ip;
                         if (property_exists($incidents, $ip)) {
                             $incidentTmp = $incidents->{$ip};
                             $incident = $incidentTmp[0];
                             $newEvent = new Event();
                             $newEvent->evidence_id = $evidenceID;
                             $newEvent->information = $incident->information;
                             $newEvent->source = $incident->source;
                             $newEvent->ticket_id = $newTicket->id;
                             $newEvent->timestamp = $incident->timestamp;
                         } else {
                             // Parser did not find any related evidence so replay it from the ticket only reason
                             // Why it happends here if DNS has changed and google report cannot be matched
                             $newEvent = new Event();
                             $newEvent->evidence_id = $evidenceID;
                             $newEvent->information = $ticket->Information;
                             $newEvent->source = $ticket->Source;
                             $newEvent->ticket_id = $newTicket->id;
                             $newEvent->timestamp = $ticket->FirstSeen;
                         }
                         // Validate the model before saving
                         $validator = Validator::make(json_decode(json_encode($newEvent), true), Event::createRules());
                         if ($validator->fails()) {
                             $this->error('DevError: Internal validation failed when saving the Event object ' . implode(' ', $validator->messages()->all()));
                             $this->exception();
                         }
                         $newEvent->save();
                     }
                 }
             }
             $this->output->progressFinish();
         } else {
             $this->info('skipping migration - phase 3 - Tickets');
         }
         if (empty($this->option('skipnotes'))) {
             $this->info('starting migration - phase 4 - Notes');
             DB::setDefaultConnection('abuseio3');
             $notes = DB::table('Notes')->get();
             DB::setDefaultConnection('mysql');
             $this->output->progressStart(count($notes));
             foreach ($notes as $note) {
                 $newNote = new Note();
                 $newNote->id = $note->ID;
                 $newNote->ticket_id = $note->ReportID;
                 $newNote->submitter = $note->Submittor;
                 $newNote->text = $note->Text;
                 $newNote->hidden = true;
                 $newNote->viewed = true;
                 $newNote->created_at = Carbon::parse($note->LastModified);
                 $newNote->updated_at = Carbon::parse($note->LastModified);
                 $validation = Validator::make($newNote->toArray(), Note::createRules());
                 if ($validation->fails()) {
                     $message = implode(' ', $validation->messages()->all());
                     $this->error('fatal error while creating contacts :' . $message);
                     $this->exception();
                 } else {
                     $newNote->save();
                 }
                 $this->output->progressAdvance();
                 echo " Working on note {$note->ID}       ";
             }
             $this->output->progressFinish();
         } else {
             $this->info('skipping migration - phase 4 - Notes');
         }
     }
     return true;
 }
Ejemplo n.º 12
0
 /**
  * Execute the command.
  *
  * @return void
  */
 public function handle()
 {
     Log::info(get_class($this) . ': ' . 'Queued worker is starting the processing of email file: ' . $this->filename);
     $rawEmail = Storage::get($this->filename);
     $parsedMail = new MimeParser();
     $parsedMail->setText($rawEmail);
     // Sanity checks
     if (empty($parsedMail->getHeader('from')) || empty($parsedMail->getMessageBody())) {
         Log::warning(get_class($this) . ': ' . 'Missing e-mail headers from and/or empty body: ' . $this->filename);
         $this->exception();
         return;
     }
     // Ignore email from our own notification address to prevent mail loops
     if (preg_match('/' . Config::get('main.notifications.from_address') . '/', $parsedMail->getHeader('from'))) {
         Log::warning(get_class($this) . ': ' . 'Loop prevention: Ignoring email from self ' . Config::get('main.notifications.from_address'));
         $this->exception();
         return;
     }
     // Start with detecting valid ARF e-mail
     $attachments = $parsedMail->getAttachments();
     $arfMail = [];
     foreach ($attachments as $attachment) {
         if ($attachment->contentType == 'message/feedback-report') {
             $arfMail['report'] = $attachment->getContent();
         }
         if ($attachment->contentType == 'message/rfc822') {
             $arfMail['evidence'] = utf8_encode($attachment->getContent());
         }
         if ($attachment->contentType == 'text/plain') {
             $arfMail['message'] = $attachment->getContent();
         }
     }
     /*
      * Sometimes the mime header does not set the main message correctly. This is ment as a fallback and will
      * use the original content body (which is basicly the same mime element). But only fallback if we actually
      * have a RFC822 message with a feedback report.
      */
     if (empty($arfMail['message']) && isset($arfMail['report']) && isset($arfMail['evidence'])) {
         $arfMail['message'] = $parsedMail->getMessageBody();
     }
     // If we do not have a complete e-mail, then we empty the perhaps partially filled arfMail
     // which is useless, hence reset to false
     if (!isset($arfMail['report']) || !isset($arfMail['evidence']) || !isset($arfMail['message'])) {
         $arfMail = false;
     }
     // Asking ParserFactory for an object based on mappings, or die trying
     $parser = ParserFactory::create($parsedMail, $arfMail);
     if ($parser !== false) {
         $parserResult = $parser->parse();
     } else {
         Log::error(get_class($this) . ': ' . ': No parser available to handle message from : ' . $parsedMail->getHeader('from') . ' with subject: ' . $parsedMail->getHeader('subject'));
         $this->exception();
         return;
     }
     if ($parserResult !== false && $parserResult['errorStatus'] === true) {
         Log::error(get_class($parser) . ': ' . ': Parser has ended with fatal errors ! : ' . $parserResult['errorMessage']);
         $this->exception();
         return;
     } else {
         Log::info(get_class($parser) . ': ' . ': Parser completed with ' . $parserResult['warningCount'] . ' warnings and collected ' . count($parserResult['data']) . ' incidents to save');
     }
     if ($parserResult['warningCount'] !== 0 && Config::get('main.emailparser.notify_on_warnings') === true) {
         Log::error(get_class($this) . ': ' . 'Configuration has warnings set as critical and ' . $parserResult['warningCount'] . ' warnings were detected. Sending alert to administrator');
         $this->exception();
         return;
     }
     /*
      * build evidence model, but wait with saving it
      **/
     $evidence = new Evidence();
     $evidence->filename = $this->filename;
     $evidence->sender = $parsedMail->getHeader('from');
     $evidence->subject = $parsedMail->getHeader('subject');
     /*
      * Call IncidentsProcess to validate, store evidence and save incidents
      */
     $incidentsProcess = new IncidentsProcess($parserResult['data'], $evidence);
     // Only continue if not empty, empty set is acceptable (exit OK)
     if (!$incidentsProcess->notEmpty()) {
         return;
     }
     // Validate the data set
     if (!$incidentsProcess->validate()) {
         $this->exception();
         return;
     }
     // Write the data set to database
     if (!$incidentsProcess->save()) {
         $this->exception();
         return;
     }
     Log::info(get_class($this) . ': ' . 'Queued worker has ended the processing of email file: ' . $this->filename);
 }
Ejemplo n.º 13
0
 /**
  * This is a ARF mail with a single incident
  *
  * @return array $reports
  */
 public function parseSpamReportArf()
 {
     $reports = [];
     //Seriously spamcop? Newlines arent in the CL specifications
     $this->arfMail['report'] = str_replace("\r", "", $this->arfMail['report']);
     preg_match_all('/([\\w\\-]+): (.*)[ ]*\\r?\\n/', $this->arfMail['report'], $regs);
     $report = array_combine($regs[1], $regs[2]);
     //Valueable information put in the body instead of the report, thnx for that Spamcop
     if (strpos($this->arfMail['message'], 'Comments from recipient') !== false) {
         preg_match("/Comments from recipient.*\\s]\n(.*)\n\n\nThis/s", str_replace(array("\r", "> "), "", $this->arfMail['message']), $match);
         $report['recipient_comment'] = str_replace("\n", " ", $match[1]);
     }
     // Add the headers from evidence into infoblob
     $parsedEvidence = new MimeParser();
     $parsedEvidence->setText($this->arfMail['evidence']);
     $headers = $parsedEvidence->getHeaders();
     foreach ($headers as $key => $value) {
         if (is_array($value) || is_object($value)) {
             foreach ($value as $index => $subvalue) {
                 $report['headers']["{$key}{$index}"] = "{$subvalue}";
             }
         } else {
             $report['headers']["{$key}"] = $value;
         }
     }
     /*
      * Sometimes Spamcop has a trouble adding the correct fields. The IP is pretty
      * normal to add. In a last attempt we will try to fetch the IP from the body ourselves
      */
     if (empty($report['Source-IP'])) {
         preg_match("/Email from (?<ip>[a-f0-9:\\.]+) \\/ " . preg_quote($report['Received-Date']) . "/s", $this->arfMail['message'], $regs);
         if (!empty($regs['ip']) && !filter_var($regs['ip'], FILTER_VALIDATE_IP) === false) {
             $report['Source-IP'] = $regs['ip'];
         }
         preg_match("/from: (?<ip>[a-f0-9:\\.]+)\r?\n?\r\n/s", $this->parsedMail->getMessageBody(), $regs);
         if (!empty($regs['ip']) && !filter_var($regs['ip'], FILTER_VALIDATE_IP) === false) {
             $report['Source-IP'] = $regs['ip'];
         }
     }
     $reports[] = $report;
     return $reports;
 }