public function execute()
{
$this->_logger->setInstanceName($this->instanceName);
$this->_pidFile = new PidFile("", $this->instanceName);
while (true) {
$startedAt = time();
$startedAt -= $startedAt % 60;
$collection = new RecordCollection(new Campaign());
if (!$collection->hasMappers()) {
Log::warning('No mappers found');
}
foreach ($collection as $campaign) {
/** @var Campaign $campaign */
if ($campaign->isDue($startedAt)) {
try {
Defero::pushCampaign($campaign->id(), $startedAt);
if (CronParser::isValid($campaign->sendAt)) {
// check average sends on scheduled
$avgEndDate = (new \DateTime())->setTimestamp($startedAt);
$avgStartDate = CronParser::prevRun($campaign->sendAt, $avgEndDate);
$avgEndDate->sub($avgStartDate->diff($avgEndDate));
$avgStartDate->setTime($avgStartDate->format('H') - 1, 0, 0);
$latestStats = MailStatistic::getCampaignStats($campaign->id(), $avgStartDate, $avgEndDate);
$diff = $avgStartDate->diff($avgEndDate);
$diffLatest = max(1, intval($diff->format('%i')) + intval($diff->format('%h') * 60) + intval($diff->format('%d') * 3600));
$latestHourly = $latestStats->sent / $diffLatest * 60;
if ($campaign->warnMin && $latestHourly < $campaign->warnMin || $campaign->warnMax && $latestHourly > $campaign->warnMax) {
Log::warning('Sending outside threshold', ['campaign' => $campaign->id(), 'average' => $latestHourly, 'warnMin' => $campaign->warnMin, 'warnMax' => $campaign->warnMax]);
}
}
} catch (\Exception $e) {
Log::error('Campaign ' . $campaign->id() . ': ' . $e->getMessage() . ' (Line: ' . $e->getLine() . ')');
}
} else {
Log::debug('Campaign ' . $campaign->id() . ' not due');
}
}
$endTime = time();
$endTime -= $endTime % 60;
if ($endTime == $startedAt) {
sleep(30);
}
}
}
public function getStats(\DateTime $from, \DateTime $to, $language = null)
{
return MailStatistic::getCampaignStats($this->id(), $from, $to, $language);
}
/**
* @param int $campaignId
* @param array[] $batch
*
* @return bool
* @throws \Exception
*/
public static function pushMessageBatch($campaignId, array $batch)
{
if (!$batch) {
return false;
}
$cacheId = 'DeferoQueueCampaign' . $campaignId;
/**
* @var Campaign $campaign
* @var Contact $contact
*/
$campaign = ExpiringEphemeralCache::getCache($cacheId, __CLASS__);
if ($campaign === null) {
$campaign = new Campaign($campaignId);
$campaign->reload();
ExpiringEphemeralCache::storeCache($cacheId, $campaign, __CLASS__, 60);
}
if (!$campaign || !$campaign->exists()) {
throw new \Exception('Campaign does not exist');
}
$campaignId = $campaign->id();
$processorsCacheId = $cacheId . ':processors';
$processors = ExpiringEphemeralCache::getCache($processorsCacheId, __CLASS__);
if ($processors === null) {
$processors = [];
$processorsConfig = Container::get(Container::CONFIG)->get('processors');
if ($campaign->processors) {
foreach ($campaign->processors as $processorData) {
$config = new Config();
$config->hydrate($processorData);
$configGroup = new ConfigGroup();
$configGroup->addConfig("process", $config);
$process = new ProcessDefinition();
$process->setProcessClass($processorsConfig->getStr($processorData->processorType));
$process->setQueueName("defero");
$process->setQueueService("queue");
$process->configure($configGroup);
$processors[] = $process;
}
} else {
throw new \Exception("Cannot queue campaign No default processors found.");
}
ExpiringEphemeralCache::storeCache($processorsCacheId, $processors, __CLASS__, 60);
}
$blacklistDomains = [];
foreach (Container::config()->get('blacklist')->getArr('domains', []) as $d) {
$blacklistDomains[] = preg_quote($d, '/');
}
$blacklistRegex = '/(' . implode('|', $blacklistDomains) . ')$/i';
//grab all user_ids from $batch, check SentEmailLog
$logKeys = [];
$keyedBatch = [];
//use this to recover $batch
$dedupe = true;
foreach ($batch as $data) {
if (isset($data['user_id'])) {
$logKeys[] = $data['user_id'] . '-' . $campaignId;
$keyedBatch[$data['user_id']] = $data;
}
if (isset($data['dedupe'])) {
$dedupe = false;
}
}
if ($logKeys && $dedupe) {
try {
//check if we sent this campaign to users today or the previous day
$yesterday = date('Y-m-d', strtotime('-1 day'));
$today = date('Y-m-d');
$result = SentEmailLog::cf()->multiGet($logKeys, [$yesterday, $today]);
foreach ($result as $key => $column) {
if (is_array($column) && (isset($column[$yesterday]) || isset($column[$today]))) {
list($userId, $cId) = explode('-', $key);
//remove user from keyedBatch because they have already
// received this campaign
unset($keyedBatch[$userId]);
\Log::info("Skipping user because they already got this campaign: [user_id:" . $data['user_id'] . ', campaign_id: ' . $campaignId . "]");
}
}
$batch = array_values($keyedBatch);
} catch (\Exception $e) {
\Log::error("Email Deduping Failed: " . $e->getMessage());
}
}
$messages = [];
foreach ($batch as $data) {
$data = array_change_key_case($data);
// check blacklist
if ($blacklistDomains && preg_match($blacklistRegex, $data['email'])) {
continue;
}
// move language here.
$userLanguage = $data['language'] = !empty($data['language']) ? $data['language'] : 'en';
$active = isset($data['campaignactive']) ? $data['campaignactive'] : $campaign->active;
$message = new ProcessMessage();
$message->setData('campaignId', $campaignId);
$message->setData('campaignActive', $active);
if (!$active) {
$message->setData('emailService', 'email_test');
} elseif ($campaign->emailService) {
$message->setData('emailService', $campaign->emailService);
} else {
$message->setData('emailService', 'email');
}
if ($campaign->replyTo) {
$message->setData('replyTo', $campaign->replyTo);
}
$message->setData('mailerTracking', $campaign->trackingType);
$message->setData('data', $data);
$languageCacheId = $cacheId . ':language:' . $userLanguage;
$msg = ExpiringEphemeralCache::getCache($languageCacheId, __CLASS__);
if ($msg === null) {
$msg = $campaign->message();
$msg->setLanguage($userLanguage);
$msg->reload();
if ($userLanguage !== 'en' && (!$msg->active || !$msg->subject || $campaign->sendType != SendType::PLAIN_TEXT && !$msg->htmlContent || $campaign->sendType != SendType::HTML_ONLY && !$msg->plainText)) {
//for non eng if html and plain but no html we shouldn't default to english
if ($campaign->sendType == SendType::HTML_AND_PLAIN && !$msg->htmlContent && $msg->plainText) {
} else {
$msg->setLanguage('en');
$msg->reload();
}
}
ExpiringEphemeralCache::storeCache($languageCacheId, $msg, __CLASS__, 60);
}
$contactId = $msg->contactId ?: $campaign->contactId;
$contactCacheId = $cacheId . ':contact:' . $contactId;
$contact = ExpiringEphemeralCache::getCache($contactCacheId, __CLASS__);
if ($contact === null) {
$contact = new Contact($contactId);
ExpiringEphemeralCache::storeCache($contactCacheId, $contact, __CLASS__, 60);
}
$data['signature'] = $contact->signature;
$message->setData('senderName', self::replaceData($contact->name, $data));
$message->setData('senderEmail', self::replaceData($contact->email, $data));
$message->setData('returnPath', self::replaceData($contact->returnPath, $data));
$message->setData('sendType', self::replaceData($campaign->sendType, $data));
$message->setData('subject', self::replaceData($msg->subject, $data));
$message->setData('plainText', self::replaceData($msg->plainText, $data));
$message->setData('htmlContent', self::replaceData($msg->htmlContent, $data, true));
foreach ($processors as $process) {
$message->addProcess($process);
}
$messages[] = serialize($message);
}
// queue
\Queue::setDefaultQueueProvider("messagequeue");
// prioritize
if ($campaign->active) {
$priority = (int) $campaign->priority;
$priority = $priority < 0 ? 1 : $priority;
} else {
$priority = 99;
}
$queueName = 'mailer.defero_messages_priority_' . $priority;
\Queue::pushBatch(new StdQueue($queueName), $messages);
// stats
try {
$hour = time();
$hour -= $hour % 3600;
$statsCf = MailStatistic::cf();
$statsCf->increment($campaignId, $hour . '|queued', count($messages));
} catch (\Exception $e) {
\Log::error('Error writing stats for campaign ' . $campaignId . ' : ' . $e->getMessage());
}
\Log::info('Queued ' . count($messages) . ' messages for Campaign ' . $campaignId);
return true;
}
public function process()
{
$userData = $this->_message->getArr('data');
$campaignActive = $this->_message->getInt('campaignActive');
$serviceName = $this->_message->getStr('emailService', $campaignActive ? 'email' : 'email_test');
$name = null;
if (isset($userData['firstname'])) {
$name = $userData['firstname'];
if (isset($userData['lastname'])) {
$name .= ' ' . $userData['lastname'];
}
}
$email = $userData['email'];
Log::info("Sending to {$name} <{$email}> using {$serviceName}");
$mailer = Email::getAccessor($serviceName);
$mailer->addRecipient($email, $name);
$mailer->setSubject($this->_message->getStr('subject'));
switch ($this->_message->getStr('sendType')) {
case SendType::PLAIN_TEXT:
$mailer->setTextBody($this->_message->getStr('plainText') ?: null);
break;
case SendType::HTML_ONLY:
$mailer->setHtmlBody($this->_message->getStr('htmlContent') ?: null);
break;
case SendType::HTML_AND_PLAIN:
$mailer->setTextBody($this->_message->getStr('plainText') ?: null);
$mailer->setHtmlBody($this->_message->getStr('htmlContent') ?: null);
break;
}
if ($mailer instanceof \Cubex\Email\Service\Mail) {
$mailer->addHeader("X-Defero-MID", uniqid(class_shortname($mailer), true));
$mailer->addHeader("X-Defero-PID", getmypid());
}
$campaignId = $this->_message->getStr('campaignId');
if ($mailer instanceof DatabaseMailer) {
$mailer->setCampaignId($campaignId);
}
$mailer->setFrom($this->_message->getStr('senderEmail'), $this->_message->getStr('senderName'));
$returnPath = $this->_message->getStr('returnPath');
if ($returnPath) {
$mailer->setSender($returnPath);
}
$replyTo = $this->_message->getStr('replyTo');
if ($replyTo) {
$mailer->setSender($replyTo);
} else {
$mailer->setSender($this->_message->getStr('senderEmail'), $this->_message->getStr('senderName'));
}
try {
$result = $mailer->send();
} catch (\Exception $e) {
Log::debug($e->getMessage());
$result = false;
}
$hour = time();
$hour -= $hour % 3600;
if (isset($userData['statskey'])) {
$brandStatsCf = MailStatisticsByBrand::cf();
$column = $hour . '|failed|' . $userData['statskey'] . '|' . $userData['language'];
if ($result !== false) {
$column = $hour . '|' . ($campaignActive ? 'sent' : 'test');
$column .= '|' . $userData['statskey'] . '|' . $userData['language'];
$brandStatsCf->increment($campaignId, $column);
} else {
$brandStatsCf->increment($campaignId, $column);
}
}
$statsCf = MailStatistic::cf();
$column = $hour . '|failed-' . $userData['language'];
if ($result !== false) {
$column = $hour . '|' . ($campaignActive ? 'sent' : 'test');
$column .= '-' . $userData['language'];
$statsCf->increment($campaignId, $column);
} else {
$statsCf->increment($campaignId, $column);
}
if (!$userData['user_id']) {
$userData['user_id'] = 0;
}
MailerLog::addLogEntry($userData['user_id'], $campaignId);
if ($result !== false) {
if ($campaignActive) {
SentEmailLog::addLogEntry($userData['user_id'], $campaignId);
}
}
return false;
}