/**
* Bootstrap the application services.
*
* @return void
*/
public function boot()
{
/*
* Provide a method to override the default configuration per environment. We only accept three named
* environments called development, testing and production. Each of these environments has a directory
* inside config where an config override can be made.
*/
$envConfig = Config::get(app()->environment());
if (!empty($envConfig) && is_array($envConfig)) {
foreach ($envConfig as $configKey => $configElement) {
$overrideConfig = $envConfig[$configKey];
$defaultConfig = Config::get($configKey);
$configWithOverrides = array_replace_recursive($defaultConfig, $overrideConfig);
Config::set($configKey, $configWithOverrides);
}
}
/*
* Were updating the timezone manually because we are updating the config later then boot. Using method from:
* vendor/laravel/framework/src/Illuminate/Foundation/Bootstrap/LoadConfiguration.php
* This enforces the timezone we use in the override config file.
*/
date_default_timezone_set(config('app.timezone'));
// Publish config files of all installed Parsers
// All parser configs we will put into the master 'parsers' tree of the config
// So we can easily walk through all of them based on the active configuration
$parserList = ParserFactory::getParsers();
$this->buildConfig($parserList, 'parser');
// Publish config files of all installed Collectors the same way
$collectorList = CollectorFactory::getCollectors();
$this->buildConfig($collectorList, 'collector');
// Publish config files of all installed Collectors the same way
$notificationList = NotificationFactory::getNotification();
$this->buildConfig($notificationList, 'notification');
}
/**
* Create and return a Parser class and it's configuration
* @param \PhpMimeMailParser\Parser $parsedMail
* @param array $arfMail
* @return object parser
*/
public static function create($parsedMail, $arfMail)
{
/**
* Loop through the parser list and try to find a match by
* validating the send or the body according to the parsers'
* configuration.
*/
$parsers = Factory::getParsers();
foreach ($parsers as $parserName) {
$parserClass = 'AbuseIO\\Parsers\\' . $parserName;
// Parser is enabled, see if we can match it's sender_map or body_map
if (config("parsers.{$parserName}.parser.enabled") === true) {
// Check validity of the 'report_file' setting before we continue
// If no report_file is used, continue w/o validation
$report_file = config("parsers.{$parserName}.parser.report_file");
if ($report_file == null || is_string($report_file) && isValidRegex($report_file)) {
$isValidReport = true;
} else {
$isValidReport = false;
Log::warning('AbuseIO\\Parsers\\Factory: ' . "The parser {$parserName} has an invalid value for 'report_file' (not a regex).");
break;
}
// Check the sender address
foreach (config("parsers.{$parserName}.parser.sender_map") as $senderMap) {
if (isValidRegex($senderMap)) {
if (preg_match($senderMap, $parsedMail->getHeader('from')) && $isValidReport) {
return new $parserClass($parsedMail, $arfMail);
}
} else {
Log::warning('AbuseIO\\Parsers\\Factory: ' . "The parser {$parserName} has an invalid value for 'sender_map' (not a regex).");
}
}
// If no valid sender is found, check the body
foreach (config("parsers.{$parserName}.parser.body_map") as $bodyMap) {
if (isValidRegex($bodyMap)) {
if (preg_match($bodyMap, $parsedMail->getMessageBody()) && $isValidReport) {
return new $parserClass($parsedMail, $arfMail);
}
if ($arfMail !== false) {
foreach ($arfMail as $mailPart) {
if (preg_match($bodyMap, $mailPart)) {
return new $parserClass($parsedMail, $arfMail);
}
}
}
} else {
Log::warning('AbuseIO\\Parsers\\Factory: ' . "The parser {$parserName} has an invalid value for 'body_map' (not a regex).");
}
}
} else {
Log::info('AbuseIO\\Parsers\\Factory: ' . "The parser {$parserName} has been disabled and will not be used for this message.");
}
}
// No valid parsers found
return false;
}
/**
* Bootstrap the application services.
*
* @return void
*/
public function boot()
{
// Publish config files of all installed Parsers
// All parser configs we will put into the master 'parsers' tree of the config
// So we can easily walk through all of them based on the active configuration
$parserList = Factory::getParsers();
foreach ($parserList as $parser) {
$basePath = base_path() . '/vendor/abuseio/parser-' . strtolower($parser) . '/config';
$parserConfig = $basePath . "/{$parser}.php";
if (File::exists($parserConfig)) {
$this->mergeConfigFrom($parserConfig, 'parsers.' . $parser);
}
$parserOverride = $basePath . '/' . app()->environment() . "/{$parser}.php";
if (File::exists($parserOverride)) {
$this->mergeConfigFrom($parserOverride, app()->environment() . '.parsers.' . $parser);
}
}
}
/**
* Execute the command
*
* @return void
*/
public function handle()
{
Log::info('(JOB ' . getmypid() . ') ' . get_class($this) . ': ' . 'Queued worker is starting the processing of email file: ' . $this->filename);
$filesystem = new Filesystem();
$rawEmail = $filesystem->get($this->filename);
$parsedMail = new MimeParser();
$parsedMail->setText($rawEmail);
// Sanity checks
if (empty($parsedMail->getHeader('from')) || empty($parsedMail->getMessageBody())) {
Log::warning('(JOB ' . getmypid() . ') ' . get_class($this) . ': ' . 'Missing e-mail headers from and/or empty body: ' . $this->filename);
$this->alertAdmin();
return;
}
// Ignore email from our own notification address to prevent mail loops
if (preg_match('/' . Config::get('main.notifications.from_address') . '/', $parsedMail->getHeader('from'))) {
Log::warning('(JOB ' . getmypid() . ') ' . get_class($this) . ': ' . 'Loop prevention: Ignoring email from self ' . Config::get('main.notifications.from_address'));
$this->alertAdmin();
return;
}
// Start with detecting valid ARF e-mail
$attachments = $parsedMail->getAttachments();
$arfMail = [];
foreach ($attachments as $attachment) {
if ($attachment->contentType == 'message/feedback-report') {
$arfMail['report'] = $attachment->getContent();
}
if ($attachment->contentType == 'message/rfc822') {
$arfMail['evidence'] = utf8_encode($attachment->getContent());
}
if ($attachment->contentType == 'text/plain') {
$arfMail['message'] = $attachment->getContent();
}
}
/*
* Sometimes the mime header does not set the main message correctly. This is ment as a fallback and will
* use the original content body (which is basicly the same mime element). But only fallback if we actually
* have a RFC822 message with a feedback report.
*/
if (empty($arfMail['message']) && isset($arfMail['report']) && isset($arfMail['evidence'])) {
$arfMail['message'] = $parsedMail->getMessageBody();
}
// If we do not have a complete e-mail, then we empty the perhaps partially filled arfMail
// which is useless, hence reset to false
if (!isset($arfMail['report']) || !isset($arfMail['evidence']) || !isset($arfMail['message'])) {
$arfMail = false;
}
// Asking ParserFactory for an object based on mappings, or die trying
$parser = ParserFactory::create($parsedMail, $arfMail);
if ($parser !== false) {
$parserResult = $parser->parse();
} else {
Log::error('(JOB ' . getmypid() . ') ' . get_class($this) . ': ' . ': No parser available to handle message from : ' . $parsedMail->getHeader('from') . ' with subject: ' . $parsedMail->getHeader('subject'));
$this->alertAdmin();
return;
}
if ($parserResult !== false && $parserResult['errorStatus'] === true) {
Log::error('(JOB ' . getmypid() . ') ' . get_class($parser) . ': ' . ': Parser has ended with fatal errors ! : ' . $parserResult['errorMessage']);
$this->alertAdmin();
return;
} else {
Log::info('(JOB ' . getmypid() . ') ' . get_class($parser) . ': ' . ': Parser completed with ' . $parserResult['warningCount'] . ' warnings and collected ' . count($parserResult['data']) . ' events to save');
}
if ($parserResult['warningCount'] !== 0 && Config::get('main.emailparser.notify_on_warnings') === true) {
Log::error('(JOB ' . getmypid() . ') ' . get_class($this) . ': ' . 'Configuration has warnings set as critical and ' . $parserResult['warningCount'] . ' warnings were detected. Sending alert to administrator');
$this->alertAdmin();
return;
}
if (count($parserResult['data']) !== 0) {
// Call validator
$validator = new EventsValidate();
$validatorResult = $validator->check($parserResult['data']);
if ($validatorResult['errorStatus'] === true) {
Log::error('(JOB ' . getmypid() . ') ' . get_class($validator) . ': ' . 'Validator has ended with errors ! : ' . $validatorResult['errorMessage']);
$this->alertAdmin();
return;
} else {
Log::info('(JOB ' . getmypid() . ') ' . get_class($validator) . ': ' . 'Validator has ended without errors');
}
/**
* save evidence into table
**/
$evidence = new Evidence();
$evidence->filename = $this->filename;
$evidence->sender = $parsedMail->getHeader('from');
$evidence->subject = $parsedMail->getHeader('subject');
$evidence->save();
/**
* call saver
**/
$saver = new EventsSave();
$saverResult = $saver->save($parserResult['data'], $evidence->id);
/**
* We've hit a snag, so we are gracefully killing ourselves
* after we contact the admin about it. EventsSave should never
* end with problems unless the mysql died while doing transactions
**/
if ($saverResult['errorStatus'] === true) {
Log::error('(JOB ' . getmypid() . ') ' . get_class($saver) . ': ' . 'Saver has ended with errors ! : ' . $saverResult['errorMessage']);
$this->alertAdmin();
return;
} else {
Log::info('(JOB ' . getmypid() . ') ' . get_class($saver) . ': ' . 'Saver has ended without errors');
}
} else {
Log::warning('(JOB ' . getmypid() . ') ' . get_class($this) . ': ' . 'Parser did not return any events therefore skipping validation and saving a empty event set');
}
Log::info('(JOB ' . getmypid() . ') ' . get_class($this) . ': ' . 'Queued worker has ended the processing of email file: ' . $this->filename);
}
/**
* Execute the command.
*
* @return void
*/
public function handle()
{
Log::info(get_class($this) . ': Queued worker is starting the processing of email file: ' . $this->filename);
$filesystem = new Filesystem();
$rawEmail = $filesystem->get($this->filename);
$parsedMail = new MimeParser();
$parsedMail->setText($rawEmail);
// Sanity checks
if (empty($parsedMail->getHeader('from')) || empty($parsedMail->getMessageBody())) {
Log::warning(get_class($this) . 'Validation failed on: ' . $this->filename);
$this->exception();
}
// Ignore email from our own notification address to prevent mail loops
if (preg_match('/' . Config::get('main.notifications.from_address') . '/', $parsedMail->getHeader('from'))) {
Log::warning(get_class($this) . 'Loop prevention: Ignoring email from self ' . Config::get('main.notifications.from_address'));
$this->exception();
}
// Start with detecting valid ARF e-mail
$attachments = $parsedMail->getAttachments();
$arfMail = [];
foreach ($attachments as $attachment) {
if ($attachment->contentType == 'message/feedback-report') {
$arfMail['report'] = $attachment->getContent();
}
if ($attachment->contentType == 'message/rfc822') {
$arfMail['evidence'] = $attachment->getContent();
}
if ($attachment->contentType == 'text/plain') {
$arfMail['message'] = $attachment->getContent();
}
}
// If we do not have a complete e-mail, then we empty the perhaps partially filled arfMail
// which is useless, hence reset to false
if (!isset($arfMail['report']) || !isset($arfMail['evidence']) || !isset($arfMail['message'])) {
$arfMail = false;
}
// Asking GetParser for an object based on mappings, or die trying
$parser = GetParser::object($parsedMail, $arfMail);
$result = false;
$events = false;
if ($parser !== false) {
$result = $parser->parse();
} else {
Log::error(get_class($this) . ': Unable to handle message from: ' . $parsedMail->getHeader('from') . ' with subject: ' . $parsedMail->getHeader('subject'));
$this->exception();
}
if ($result !== false && $result['errorStatus'] !== true) {
Log::info(get_class($parser) . ': Parser as ended without errors. Collected ' . count($result['data']) . ' events to save');
$events = $result['data'];
} else {
Log::error(get_class($parser) . ': Parser as ended with errors ! : ' . $result['errorMessage']);
$this->exception();
}
// call validater
$validator = new EventsValidate($events);
$return = $validator->handle();
if ($return['errorStatus'] === false) {
Log::error(get_class($validator) . ': Validator as ended with errors ! : ' . $result['errorMessage']);
$this->exception();
} else {
Log::info(get_class($validator) . ': Validator as ended without errors');
}
// save evidence into table
$evidence = new Evidence();
$evidence->filename = $this->filename;
$evidence->sender = $parsedMail->getHeader('from');
$evidence->subject = $parsedMail->getHeader('subject');
$evidence->save();
// call saver
$saver = new EventsSave($events, $evidence->id);
$return = $saver->handle();
if ($return['errorStatus'] === false) {
Log::error(get_class($saver) . ': Saver as ended with errors ! : ' . $result['errorMessage']);
$this->exception();
} else {
Log::info(get_class($saver) . ': Saver as ended without errors');
}
}
/**
* Execute the console command.
*
* @return bool
*/
public function handle()
{
$account = Account::getSystemAccount();
if (empty($this->option('start')) && empty($this->option('prepare')) && empty($this->option('clean'))) {
$this->error('You need to either prepare or start the migration. try --help');
die;
}
if (!empty($this->option('clean'))) {
$this->warn('This will remove all data from the database except prepared data. Do not run ' . 'this in producion and only when a migration has gone bad and you want to restart it');
if ($this->confirm('Do you wish to continue? [y|N]')) {
$this->info('starting clean up');
DB::statement('SET foreign_key_checks=0');
Note::truncate();
Event::truncate();
Ticket::truncate();
Netblock::truncate();
Contact::truncate();
DB::statement('SET foreign_key_checks=1');
} else {
$this->info('cancelled clean up');
}
}
/*
* Combine the use of start/stop and threading here for usage with prepare and start of migration
*/
$startFrom = $this->option('startfrom');
$endWith = $this->option('endwith');
if (!empty($this->option('prepare')) || !empty($this->option('start'))) {
if (!empty($this->option('threaded'))) {
if (empty($this->option('threadid')) || empty($this->option('threadsize'))) {
$this->error('threadid and threadsize are required to calculate this threads start/stop ID');
die;
}
$this->info("*** using threaded mode, instance ID {$this->option('threadid')}");
$startFrom = $this->option('threadid') * $this->option('threadsize') - $this->option('threadsize') + 1;
$endWith = $startFrom + $this->option('threadsize') - 1;
$this->info("*** starting with ticket {$startFrom} " . "and ending with ticket {$endWith} ");
}
}
if (!empty($this->option('prepare'))) {
$this->info('building required evidence cache files');
$filesystem = new Filesystem();
$path = storage_path() . '/migration/';
umask(07);
if (!$filesystem->isDirectory($path)) {
// If a datefolder does not exist, then create it or die trying
if (!$filesystem->makeDirectory($path, 0770)) {
$this->error('Unable to create directory: ' . $path);
$this->exception();
}
if (!is_dir($path)) {
$this->error('Path vanished after write: ' . $path);
$this->exception();
}
chgrp($path, config('app.group'));
}
DB::setDefaultConnection('abuseio3');
$evidenceRows = DB::table('Evidence')->where('id', '>=', $startFrom)->where('id', '<=', $endWith);
$migrateCount = $evidenceRows->count();
$evidences = $evidenceRows->get();
$this->output->progressStart($migrateCount);
// If there are now rows to do, advance to complete
if ($migrateCount === 0) {
$this->output->progressAdvance();
echo ' nothing to do, because there are no records in this selection';
}
foreach ($evidences as $evidence) {
$this->output->progressAdvance();
// Before we go into an error, lets see if this evidence was even linked to any ticket at all
// If not we can ignore the error and just smile and wave
$evidenceLinks = DB::table('EvidenceLinks')->where('EvidenceID', '=', $evidence->ID)->get();
if (count($evidenceLinks) === 0) {
echo " skipping unlinked evidence ID {$evidence->ID} ";
continue;
}
$filename = $path . "evidence_id_{$evidence->ID}.data";
if (is_file($filename)) {
// Check weither the evidence was actually created in the database
DB::setDefaultConnection('mysql');
$evidenceCheck = Evidence::where('id', $evidence->ID);
if ($evidenceCheck->count() !== 1) {
$this->error("file {$filename} exists however not in database. try deleting it to retry");
die;
}
DB::setDefaultConnection('abuseio3');
continue;
} else {
echo " working on evidence ID {$evidence->ID} ";
}
$rawEmail = $evidence->Data;
$parsedMail = new MimeParser();
$parsedMail->setText($rawEmail);
// Start with detecting valid ARF e-mail
$attachments = $parsedMail->getAttachments();
$arfMail = [];
foreach ($attachments as $attachment) {
if ($attachment->contentType == 'message/feedback-report') {
$arfMail['report'] = $attachment->getContent();
}
if ($attachment->contentType == 'message/rfc822') {
$arfMail['evidence'] = utf8_encode($attachment->getContent());
}
if ($attachment->contentType == 'text/plain') {
$arfMail['message'] = $attachment->getContent();
}
}
if (empty($arfMail['message']) && isset($arfMail['report']) && isset($arfMail['evidence'])) {
$arfMail['message'] = $parsedMail->getMessageBody();
}
// If we do not have a complete e-mail, then we empty the perhaps partially filled arfMail
// which is useless, hence reset to false
if (!isset($arfMail['report']) || !isset($arfMail['evidence']) || !isset($arfMail['message'])) {
$arfMail = false;
}
// Asking ParserFactory for an object based on mappings, or die trying
$parser = ParserFactory::create($parsedMail, $arfMail);
if ($parser !== false) {
$parserResult = $parser->parse();
} else {
$this->error('No parser available to handle message ' . $evidence->ID . ' from : ' . $evidence->Sender . ' with subject: ' . $evidence->Subject);
continue;
}
if ($parserResult !== false && $parserResult['errorStatus'] === true) {
$this->error('Parser has ended with fatal errors ! : ' . $parserResult['errorMessage']);
$this->exception();
}
if ($parserResult['warningCount'] !== 0) {
$this->error('Configuration has warnings set as critical and ' . $parserResult['warningCount'] . ' warnings were detected.');
//var_dump($rawEmail);
$this->exception();
}
// Write the evidence into the archive
$evidenceWrite = new EvidenceSave();
$evidenceData = $rawEmail;
$evidenceFile = $evidenceWrite->save($evidenceData);
// Save the file reference into the database
$evidenceSave = new Evidence();
$evidenceSave->id = $evidence->ID;
$evidenceSave->filename = $evidenceFile;
$evidenceSave->sender = $parsedMail->getHeader('from');
$evidenceSave->subject = $parsedMail->getHeader('subject');
$incidentsProcess = new IncidentsProcess($parserResult['data'], $evidenceSave);
/*
* Because google finds it 'obvious' not to include the IP address relating to abuse
* the IP field might now be empty with reparsing if the domain/label does not resolve
* anymore. For these cases we need to lookup the ticket that was linked to the evidence
* match the domain and retrieve its IP.
*/
foreach ($parserResult['data'] as $index => $incident) {
if ($incident->source == 'Google Safe Browsing' && $incident->domain != false && $incident->ip == '127.0.0.1') {
// Get the list of tickets related to this evidence
$evidenceLinks = DB::table('EvidenceLinks')->where('EvidenceID', '=', $evidence->ID)->get();
// For each ticket check if the domain name is matching the evidence we need to update
foreach ($evidenceLinks as $evidenceLink) {
$ticket = DB::table('Reports')->where('ID', '=', $evidenceLink->ReportID)->first();
if ($ticket->Domain == $incident->domain) {
$incident->ip = $ticket->IP;
}
}
// Update the original object by overwriting it
$parserResult['data'][$index] = $incident;
}
}
// Only continue if not empty, empty set is acceptable (exit OK)
if (!$incidentsProcess->notEmpty()) {
$this->warn("No evidence build, no results from parser for {$evidence->ID}");
continue;
}
// Validate the data set
if (!$incidentsProcess->validate()) {
$this->error('Validation failed of object.');
$this->exception();
}
$incidents = [];
foreach ($parserResult['data'] as $incident) {
$incidents[$incident->ip][] = $incident;
}
DB::setDefaultConnection('mysql');
$evidenceSave->save();
DB::setDefaultConnection('abuseio3');
$output = ['evidenceId' => $evidence->ID, 'evidenceData' => $evidence->Data, 'incidents' => $incidents, 'newId' => $evidenceSave->id];
if ($filesystem->put($filename, json_encode($output)) === false) {
$this->error('Unable to write file: ' . $filename);
return false;
}
}
$this->output->progressFinish();
}
if (!empty($this->option('start'))) {
if (empty($this->option('skipcontacts'))) {
$this->info('starting migration - phase 1 - contact data');
DB::setDefaultConnection('abuseio3');
$customers = DB::table('Customers')->get();
DB::setDefaultConnection('mysql');
$this->output->progressStart(count($customers));
foreach ($customers as $customer) {
$newContact = new Contact();
$newContact->reference = $customer->Code;
$newContact->name = $customer->Name;
$newContact->email = $customer->Contact;
$newContact->auto_notify = $customer->AutoNotify;
$newContact->enabled = 1;
$newContact->account_id = $account->id;
$newContact->created_at = Carbon::parse($customer->LastModified);
$newContact->updated_at = Carbon::parse($customer->LastModified);
$validation = Validator::make($newContact->toArray(), Contact::createRules());
if ($validation->fails()) {
$message = implode(' ', $validation->messages()->all());
$this->error('fatal error while creating contacts :' . $message);
$this->exception();
} else {
$newContact->save();
}
$this->output->progressAdvance();
echo " Working on contact {$customer->Code} ";
}
$this->output->progressFinish();
} else {
$this->info('skipping migration - phase 1 - contact data');
}
if (empty($this->option('skipnetblocks'))) {
$this->info('starting migration - phase 2 - netblock data');
DB::setDefaultConnection('abuseio3');
$netblocks = DB::table('Netblocks')->get();
DB::setDefaultConnection('mysql');
$this->output->progressStart(count($netblocks));
foreach ($netblocks as $netblock) {
$contact = FindContact::byId($netblock->CustomerCode);
if ($contact->reference != $netblock->CustomerCode) {
$this->error('Contact lookup failed, mismatched results');
$this->{$this}->exception();
}
$newNetblock = new Netblock();
$newNetblock->first_ip = long2ip($netblock->begin_in);
$newNetblock->last_ip = long2ip($netblock->end_in);
$newNetblock->description = 'Imported from previous AbuseIO version which did not include a description';
$newNetblock->contact_id = $contact->id;
$newNetblock->enabled = 1;
$newNetblock->created_at = Carbon::parse($netblock->LastModified);
$newNetblock->updated_at = Carbon::parse($netblock->LastModified);
$validation = Validator::make($newNetblock->toArray(), Netblock::createRules($newNetblock));
if ($validation->fails()) {
$message = implode(' ', $validation->messages()->all());
$this->error('fatal error while creating contacts :' . $message);
$this->exception();
} else {
$newNetblock->save();
}
$this->output->progressAdvance();
echo ' Working on netblock ' . long2ip($netblock->begin_in) . ' ';
}
$this->output->progressFinish();
} else {
$this->info('skipping migration - phase 2 - netblock data');
}
if (empty($this->option('skiptickets'))) {
$this->info('starting migration - phase 3 - ticket and evidence data');
DB::setDefaultConnection('abuseio3');
$ticketRows = DB::table('Reports')->where('id', '>=', $startFrom)->where('id', '<=', $endWith);
$migrateCount = $ticketRows->count();
$tickets = $ticketRows->get();
DB::setDefaultConnection('mysql');
$this->output->progressStart($migrateCount);
// If there are now rows to do, advance to complete
if ($migrateCount === 0) {
$this->output->progressAdvance();
echo ' nothing to do, because there are no records in this selection';
}
foreach ($tickets as $ticket) {
// Get the list of evidence ID's related to this ticket
DB::setDefaultConnection('abuseio3');
$evidenceLinks = DB::table('EvidenceLinks')->where('ReportID', '=', $ticket->ID)->get();
DB::setDefaultConnection('mysql');
// DO NOT REMOVE! Legacy versions (1.0 / 2.0) have imports without evidence.
// These dont have any linked evidence and will require a manual building of evidence
// for now we ignore them. This will not affect any 3.x installations
if ($ticket->CustomerName == 'Imported from AbuseReporter' || !empty(json_decode($ticket->Information)->importnote)) {
// Manually build the evidence
$this->output->progressAdvance();
echo " Working on events from ticket {$ticket->ID}";
$this->replayTicket($ticket, $account);
continue;
}
if (count($evidenceLinks) != (int) $ticket->ReportCount) {
// Count does not match, known 3.0 limitation related to not always saving all the data
// so we will do a little magic to fix that
$this->output->progressAdvance();
echo " Working on events from ticket {$ticket->ID}";
$this->replayTicket($ticket, $account);
continue;
} else {
// Just work as normal
$this->output->progressAdvance();
echo " Working on events from ticket {$ticket->ID}";
$newTicket = $this->createTicket($ticket, $account);
// Create all the events
foreach ($evidenceLinks as $evidenceLink) {
$path = storage_path() . '/migration/';
$filename = $path . "evidence_id_{$evidenceLink->EvidenceID}.data";
if (!is_file($filename)) {
$this->error('missing cache file ');
$this->exception();
}
$evidence = json_decode(file_get_contents($filename));
$evidenceID = (int) $evidence->evidenceId;
$incidents = $evidence->incidents;
// Yes we only grab nr 0 from the array, because that is what the old aggregator did
// which basicly ignored a few incidents because they werent considered unique (which
// they were with the domain name)
$ip = $newTicket->ip;
if (property_exists($incidents, $ip)) {
$incidentTmp = $incidents->{$ip};
$incident = $incidentTmp[0];
$newEvent = new Event();
$newEvent->evidence_id = $evidenceID;
$newEvent->information = $incident->information;
$newEvent->source = $incident->source;
$newEvent->ticket_id = $newTicket->id;
$newEvent->timestamp = $incident->timestamp;
} else {
// Parser did not find any related evidence so replay it from the ticket only reason
// Why it happends here if DNS has changed and google report cannot be matched
$newEvent = new Event();
$newEvent->evidence_id = $evidenceID;
$newEvent->information = $ticket->Information;
$newEvent->source = $ticket->Source;
$newEvent->ticket_id = $newTicket->id;
$newEvent->timestamp = $ticket->FirstSeen;
}
// Validate the model before saving
$validator = Validator::make(json_decode(json_encode($newEvent), true), Event::createRules());
if ($validator->fails()) {
$this->error('DevError: Internal validation failed when saving the Event object ' . implode(' ', $validator->messages()->all()));
$this->exception();
}
$newEvent->save();
}
}
}
$this->output->progressFinish();
} else {
$this->info('skipping migration - phase 3 - Tickets');
}
if (empty($this->option('skipnotes'))) {
$this->info('starting migration - phase 4 - Notes');
DB::setDefaultConnection('abuseio3');
$notes = DB::table('Notes')->get();
DB::setDefaultConnection('mysql');
$this->output->progressStart(count($notes));
foreach ($notes as $note) {
$newNote = new Note();
$newNote->id = $note->ID;
$newNote->ticket_id = $note->ReportID;
$newNote->submitter = $note->Submittor;
$newNote->text = $note->Text;
$newNote->hidden = true;
$newNote->viewed = true;
$newNote->created_at = Carbon::parse($note->LastModified);
$newNote->updated_at = Carbon::parse($note->LastModified);
$validation = Validator::make($newNote->toArray(), Note::createRules());
if ($validation->fails()) {
$message = implode(' ', $validation->messages()->all());
$this->error('fatal error while creating contacts :' . $message);
$this->exception();
} else {
$newNote->save();
}
$this->output->progressAdvance();
echo " Working on note {$note->ID} ";
}
$this->output->progressFinish();
} else {
$this->info('skipping migration - phase 4 - Notes');
}
}
return true;
}
/**
* Execute the command.
*
* @return void
*/
public function handle()
{
Log::info(get_class($this) . ': ' . 'Queued worker is starting the processing of email file: ' . $this->filename);
$rawEmail = Storage::get($this->filename);
$parsedMail = new MimeParser();
$parsedMail->setText($rawEmail);
// Sanity checks
if (empty($parsedMail->getHeader('from')) || empty($parsedMail->getMessageBody())) {
Log::warning(get_class($this) . ': ' . 'Missing e-mail headers from and/or empty body: ' . $this->filename);
$this->exception();
return;
}
// Ignore email from our own notification address to prevent mail loops
if (preg_match('/' . Config::get('main.notifications.from_address') . '/', $parsedMail->getHeader('from'))) {
Log::warning(get_class($this) . ': ' . 'Loop prevention: Ignoring email from self ' . Config::get('main.notifications.from_address'));
$this->exception();
return;
}
// Start with detecting valid ARF e-mail
$attachments = $parsedMail->getAttachments();
$arfMail = [];
foreach ($attachments as $attachment) {
if ($attachment->contentType == 'message/feedback-report') {
$arfMail['report'] = $attachment->getContent();
}
if ($attachment->contentType == 'message/rfc822') {
$arfMail['evidence'] = utf8_encode($attachment->getContent());
}
if ($attachment->contentType == 'text/plain') {
$arfMail['message'] = $attachment->getContent();
}
}
/*
* Sometimes the mime header does not set the main message correctly. This is ment as a fallback and will
* use the original content body (which is basicly the same mime element). But only fallback if we actually
* have a RFC822 message with a feedback report.
*/
if (empty($arfMail['message']) && isset($arfMail['report']) && isset($arfMail['evidence'])) {
$arfMail['message'] = $parsedMail->getMessageBody();
}
// If we do not have a complete e-mail, then we empty the perhaps partially filled arfMail
// which is useless, hence reset to false
if (!isset($arfMail['report']) || !isset($arfMail['evidence']) || !isset($arfMail['message'])) {
$arfMail = false;
}
// Asking ParserFactory for an object based on mappings, or die trying
$parser = ParserFactory::create($parsedMail, $arfMail);
if ($parser !== false) {
$parserResult = $parser->parse();
} else {
Log::error(get_class($this) . ': ' . ': No parser available to handle message from : ' . $parsedMail->getHeader('from') . ' with subject: ' . $parsedMail->getHeader('subject'));
$this->exception();
return;
}
if ($parserResult !== false && $parserResult['errorStatus'] === true) {
Log::error(get_class($parser) . ': ' . ': Parser has ended with fatal errors ! : ' . $parserResult['errorMessage']);
$this->exception();
return;
} else {
Log::info(get_class($parser) . ': ' . ': Parser completed with ' . $parserResult['warningCount'] . ' warnings and collected ' . count($parserResult['data']) . ' incidents to save');
}
if ($parserResult['warningCount'] !== 0 && Config::get('main.emailparser.notify_on_warnings') === true) {
Log::error(get_class($this) . ': ' . 'Configuration has warnings set as critical and ' . $parserResult['warningCount'] . ' warnings were detected. Sending alert to administrator');
$this->exception();
return;
}
/*
* build evidence model, but wait with saving it
**/
$evidence = new Evidence();
$evidence->filename = $this->filename;
$evidence->sender = $parsedMail->getHeader('from');
$evidence->subject = $parsedMail->getHeader('subject');
/*
* Call IncidentsProcess to validate, store evidence and save incidents
*/
$incidentsProcess = new IncidentsProcess($parserResult['data'], $evidence);
// Only continue if not empty, empty set is acceptable (exit OK)
if (!$incidentsProcess->notEmpty()) {
return;
}
// Validate the data set
if (!$incidentsProcess->validate()) {
$this->exception();
return;
}
// Write the data set to database
if (!$incidentsProcess->save()) {
$this->exception();
return;
}
Log::info(get_class($this) . ': ' . 'Queued worker has ended the processing of email file: ' . $this->filename);
}
