/**
* Execute the console command.
*
* @return boolean
*/
public function handle()
{
// TODO: #AIO-22 Create housekeeping - Walk thru all collectors to gather information.
// TODO: Extra: Collectors should be kicked into a queue, only if there isn't one running yet with the same name
/*
* Checks for beanstalk queue
*/
// TODO: Somehow check howlong a job is running? Or self-kill it in 1 hour?
/*
* Fire an test jobs into the abuseio queue selected.
* Handling of the result is done at the QueueTest->failed() method
*/
// Todo how to get a list of queues, make it fixed?
//$this->dispatch(new QueueTest($queue));
/*
* Check for any kind of failed jobs
*/
$jobs = $this->getFailedJobs();
$jobsMask = "|%-8.8s |%-20.20s |%-35.35s |%-35.35s |%-30.30s |" . PHP_EOL . PHP_EOL;
$jobsList[] = sprintf($jobsMask, 'ID', 'Connection', 'Queue', 'Class', 'Failed At');
if (count($jobs) != 0) {
foreach ($jobs as $job) {
if (count($job) == 5) {
$jobsList[] = sprintf($jobsMask, $job[0], $job[1], $job[2], $job[3], $job[4]);
}
}
if (count($jobsList) > 2) {
AlertAdmin::send("Alert: There are " . count($jobs) . " jobs in the queue that have failed:" . PHP_EOL . PHP_EOL . implode(PHP_EOL, $jobsList));
}
}
/*
* Walk thru all tickets to see which need closing
*/
if (config('main.housekeeping.tickets_close_after') !== false) {
$this->ticketsClosing();
}
/*
* Walk thru mailarchive to see which need pruning
*/
if (config('main.housekeeping.mailarchive_remove_after') !== false) {
$this->mailarchivePruning();
}
/*
* Send out all notifications by calling the housekeeper notifications command
*/
if ($this->option('noNotifications') !== true) {
$this->sendNotifications();
}
return true;
}
/**
* We've hit a snag, so we are gracefully killing ourselves after we contact the admin about it.
*
* @param string $rawEmail
* @return mixed
*/
protected function exception($rawEmail)
{
// This only bounces with config errors or problems with installations where we cannot accept
// the email at all. In normal cases the bounce will be handled within EmailProcess::()
Log::error('(JOB ' . getmypid() . ') ' . get_class($this) . ': ' . 'Email receiver is ending with errors. The received e-mail will be bounced to the admin for investigation');
AlertAdmin::send('AbuseIO was not able to receive an incoming message. This message is attached to this email.', ['failed_message.eml' => $rawEmail]);
}
/**
* alert administrator when problems happens. We will add the received message as attachment or bounce the original
*
* @return void
*/
protected function alertAdmin()
{
// we have $this->filename and $this->rawMail
// and this Config::get('main.emailparser.fallback_mail')
Log::error('(JOB ' . getmypid() . ') ' . get_class($this) . ': ' . 'Email processor ending with errors. The received e-mail will be deleted from ' . 'archive and bounced to the admin for investigation');
AlertAdmin::send('AbuseIO was not able to process an incoming message. This message is attached to this email.', ['failed_message.eml' => @file_get_contents($this->filename)]);
// Delete the evidence file as we are not using it.
$filesystem = new Filesystem();
$filesystem->delete($this->filename);
}
/**
* Walk thru all jobs and queues to make sure they are working, including firing a testjob at them
*
* @return boolean
*/
private function checkQueues()
{
$jobLimit = new Carbon('1 hour ago');
$hangs = [];
foreach (config('queue.queues') as $queue) {
/*
* Fire an test jobs into the abuseio queue selected.
* Handling of the result is done at the QueueTest->failed() method
*/
$this->dispatch(new QueueTest($queue));
/*
* Check all created jobs not to be older then 1 hour
*/
$jobs = Job::where('queue', '=', $queue)->get();
foreach ($jobs as $job) {
$created = $job->created_at;
if ($jobLimit->gt($created)) {
$hangs[] = $job;
}
}
}
/*
* Send alarm on hanging jobs
*/
if (count($hangs) != 0) {
AlertAdmin::send("Alert: There are " . count($hangs) . " jobs that are stuck:" . PHP_EOL . PHP_EOL . implode(PHP_EOL, $hangs));
}
/*
* Check for any kind of failed jobs, if any found start alarm bells
*/
$failed = $this->laravel['queue.failer']->all();
if (count($failed) != 0) {
AlertAdmin::send("Alert: There are " . count($failed) . " jobs that have failed:" . PHP_EOL . PHP_EOL . implode(PHP_EOL, $failed));
}
if (count($failed) != 0 || count($hangs) != 0) {
return false;
}
return true;
}
/**
* alert administrator when problems happens. We will add the received message as attachment or bounce the original.
*
* @return void
*/
protected function exception()
{
// we have $this->filename and $this->rawMail
// and this Config::get('main.emailparser.fallback_mail')
Log::error(get_class($this) . ': ' . 'Email processor ending with errors. The received e-mail will be deleted from ' . 'archive and bounced to the admin for investigation');
$fileContents = null;
if (Storage::exists($this->filename)) {
$fileContents = Storage::get($this->filename);
}
AlertAdmin::send('AbuseIO was not able to process an incoming message. This message is attached to this email.', ['failed_message.eml' => $fileContents]);
// Delete the evidence file as we are not using it.
Storage::delete($this->filename);
}
/**
* alert administrator when problems happens. We will add the received message as attachment or bounce the original.
*
* @return void
*/
protected function exception()
{
Log::error(get_class($this) . ': ' . 'Collector processor ending with errors.');
AlertAdmin::send('AbuseIO was not able to process a collection. This the logs for PID:' . getmypid());
}
/**
* Walk thru all jobs and queues to make sure they are working, including firing a testjob at them.
*
* @return bool
*/
private function checkQueues()
{
Log::info(get_class($this) . ': Housekeeper is starting queue checks');
$jobLimit = new Carbon('1 hour ago');
$hangs = [];
foreach (config('queue.queues') as $queue) {
/*
* Fire an test jobs into the abuseio queue selected.
* Handling of the result is done at the QueueTest->failed() method
*/
$this->dispatch(new QueueTest($queue));
/*
* Check all created jobs not to be older then 1 hour
*/
$jobs = Job::where('queue', '=', $queue)->get();
foreach ($jobs as $job) {
$created = $job->created_at;
if ($jobLimit->gt($created)) {
$hangs[] = $job;
}
}
}
/*
* Send alarm on hanging jobs
*/
$hangCount = count($hangs);
if ($hangCount != 0) {
Log::warning(get_class($this) . ": Housekeeper detected {$hangCount} jobs that are stuck in one or more queues!");
if (config('main.housekeeping.enable_queue_problem_alerts')) {
AlertAdmin::send("Alert: There are {$hangCount} jobs that are stuck:" . PHP_EOL . PHP_EOL . implode(PHP_EOL, $hangs));
}
}
/*
* Check for any kind of failed jobs, if any found start alarm bells
*/
$failed = $this->laravel['queue.failer']->all();
$failedCount = count($failed);
if ($failedCount != 0) {
// Reset object to string for reporting
foreach ($failed as $key => $job) {
$failed[$key] = implode(' ', get_object_vars($job));
}
Log::warning(get_class($this) . ": Housekeeper detected failed {$failedCount} jobs which need to be handled!");
if (config('main.housekeeping.enable_queue_problem_alerts')) {
AlertAdmin::send("Alert: There are {$failedCount} jobs that have failed:" . PHP_EOL . PHP_EOL . implode(PHP_EOL, $failed));
}
}
if ($hangCount != 0 || $failedCount != 0) {
return false;
}
return true;
}
