public function runProcess(IProcessMessage $message, IProcessDefinition $process, $taskId) { $this->_queueDelays[$taskId] = 0; $class = $process->getProcessClass(); if (class_exists($class)) { $ruleClass = 'Qubes\\Defero\\Transport\\IMessageProcessor'; /* * replaced below with faster implementation: * if(in_array($ruleClass, class_implements($class)))) */ if (is_subclass_of($class, $ruleClass)) { $proc = new $class($message); } else { $proc = new $class(); } if ($proc instanceof IConfigurable) { $proc->configure($process->getConfig()); } if ($proc instanceof IRule) { if ($proc->canProcess()) { if ($proc instanceof IDeliveryRule) { $this->_queueDelays[$taskId] = (int) $proc->getSendDelay(); Log::debug("Setting queue delay to " . $this->_queueDelays[$taskId]); } return true; } return false; } else { if ($proc instanceof IProcess) { return $proc->process(); } } } return false; }
/** * @return int */ public function execute() { $this->_logger->setInstanceName($this->instanceName); $this->_pidFile = new PidFile("", $this->instanceName); echo Shell::colourText((new Figlet("speed"))->render("Defero"), Shell::COLOUR_FOREGROUND_GREEN); echo "\n"; Log::debug("Setting Default Queue Provider to " . $this->queueService); Queue::setDefaultQueueProvider($this->queueService); $queue = Queue::getAccessor(); if ($queue instanceof DatabaseQueue) { $instance = gethostname(); if ($this->instanceName) { $instance .= ':' . $this->instanceName; } $queue->setOwnKey($instance); } $priority = (int) $this->priority; if (in_array($priority, [1, 5, 10, 99])) { $this->queueName .= $priority; } else { throw new \Exception("Invalid priority. Supported values: 1 , 5, 10, 99"); } Log::info("Starting to consume queue " . $this->queueName); $queue->consume(new StdQueue($this->queueName), new CampaignConsumer()); Log::info("Exiting Defero Processor"); }
public function init($initialiser = null) { EventManager::listen(EventManager::CUBEX_QUERY, function (IEvent $e) { if (CUBEX_CLI) { Log::debug("Query: " . $e->getStr("query")); } else { var_dump($e->getStr("query")); } }); }
public function execute() { $this->_logger->setInstanceName($this->instanceName); $this->_pidFile = new PidFile("", $this->instanceName); while (true) { $startedAt = time(); $startedAt -= $startedAt % 60; $collection = new RecordCollection(new Campaign()); if (!$collection->hasMappers()) { Log::warning('No mappers found'); } foreach ($collection as $campaign) { /** @var Campaign $campaign */ if ($campaign->isDue($startedAt)) { try { Defero::pushCampaign($campaign->id(), $startedAt); if (CronParser::isValid($campaign->sendAt)) { // check average sends on scheduled $avgEndDate = (new \DateTime())->setTimestamp($startedAt); $avgStartDate = CronParser::prevRun($campaign->sendAt, $avgEndDate); $avgEndDate->sub($avgStartDate->diff($avgEndDate)); $avgStartDate->setTime($avgStartDate->format('H') - 1, 0, 0); $latestStats = MailStatistic::getCampaignStats($campaign->id(), $avgStartDate, $avgEndDate); $diff = $avgStartDate->diff($avgEndDate); $diffLatest = max(1, intval($diff->format('%i')) + intval($diff->format('%h') * 60) + intval($diff->format('%d') * 3600)); $latestHourly = $latestStats->sent / $diffLatest * 60; if ($campaign->warnMin && $latestHourly < $campaign->warnMin || $campaign->warnMax && $latestHourly > $campaign->warnMax) { Log::warning('Sending outside threshold', ['campaign' => $campaign->id(), 'average' => $latestHourly, 'warnMin' => $campaign->warnMin, 'warnMax' => $campaign->warnMax]); } } } catch (\Exception $e) { Log::error('Campaign ' . $campaign->id() . ': ' . $e->getMessage() . ' (Line: ' . $e->getLine() . ')'); } } else { Log::debug('Campaign ' . $campaign->id() . ' not due'); } } $endTime = time(); $endTime -= $endTime % 60; if ($endTime == $startedAt) { sleep(30); } } }
/** * @return int */ public function execute() { $this->_logger->setInstanceName($this->instanceName); $this->_pidFile = new PidFile("", $this->instanceName); echo Shell::colourText((new Figlet("speed"))->render("Defero"), Shell::COLOUR_FOREGROUND_GREEN); echo "\n"; Log::debug("Setting Default Queue Provider to " . $this->queueService); Queue::setDefaultQueueProvider($this->queueService); $queue = Queue::getAccessor(); if ($queue instanceof DatabaseQueue) { $instance = gethostname(); if ($this->instanceName) { $instance .= ':' . $this->instanceName; } $queue->setOwnKey($instance); } Log::info("Starting to consume queue " . $this->queueName); $queue->consume(new StdQueue($this->queueName), new CampaignQueueConsumer()); Log::info("Exiting Defero Processor"); }
private function _recalculateBatchSize() { if (count($this->_batchTimes) < $this->_bufferSize || $this->_fixedSize) { return; } while (count($this->_batchTimes) > $this->_bufferSize) { array_shift($this->_batchTimes); } $avgTime = array_sum($this->_batchTimes) / count($this->_batchTimes); $newBatchSize = $this->_currentBatchSize; if ($avgTime < $this->_minBatchTime) { // Batches are processing too fast. Increase the batch size if possible. $diff = $this->_minBatchTime - $avgTime; $proportion = $diff / $avgTime; $batchIncrease = $this->_currentBatchSize * $proportion; $newBatchSize = round($this->_currentBatchSize + $batchIncrease); if ($newBatchSize > $this->_maxBatchSize) { $newBatchSize = $this->_maxBatchSize; } } else { if ($avgTime > $this->_maxBatchTime) { // Batches are processing too slowly. Reduce the batch size if possible. $diff = $avgTime - $this->_maxBatchTime; $proportion = $diff / $avgTime; $batchDecrease = $this->_currentBatchSize * $proportion; $newBatchSize = round($this->_currentBatchSize - $batchDecrease); if ($newBatchSize < $this->_minBatchSize) { $newBatchSize = $this->_minBatchSize; } } } if ($newBatchSize != $this->_currentBatchSize) { Log::info(sprintf('BatchSizeTuner: Average batch time: %d seconds. Changing batch size from %d to %d', $avgTime, $this->_currentBatchSize, $newBatchSize)); $this->_currentBatchSize = $newBatchSize; $this->reset(); } else { Log::debug('BatchSizeTuner: Average batch time: ' . $avgTime . ' seconds, batch size=' . $this->_currentBatchSize); } }
/** * @param TokenRange[] $ranges */ private function _insertMultipleRanges($ranges) { $db = TokenRange::conn(); $tableName = (new TokenRange())->getTableName(); $query = ParseQuery::parse($db, 'INSERT INTO %T (startToken, endToken, randomKey, createdAt, updatedAt) VALUES ', $tableName); $data = []; foreach ($ranges as $range) { $nowStr = DateTimeHelper::formattedDateFromAnything(time()); $data[] = ParseQuery::parse($db, "(%s, %s, %d, %s, %s)", $range->startToken, $range->endToken, $range->randomKey, $nowStr, $nowStr); } $query .= implode(", ", $data); Log::debug($query); $db->query($query); }
public function process() { $userData = $this->_message->getArr('data'); $campaignActive = $this->_message->getInt('campaignActive'); $serviceName = $this->_message->getStr('emailService', $campaignActive ? 'email' : 'email_test'); $name = null; if (isset($userData['firstname'])) { $name = $userData['firstname']; if (isset($userData['lastname'])) { $name .= ' ' . $userData['lastname']; } } $email = $userData['email']; Log::info("Sending to {$name} <{$email}> using {$serviceName}"); $mailer = Email::getAccessor($serviceName); $mailer->addRecipient($email, $name); $mailer->setSubject($this->_message->getStr('subject')); switch ($this->_message->getStr('sendType')) { case SendType::PLAIN_TEXT: $mailer->setTextBody($this->_message->getStr('plainText') ?: null); break; case SendType::HTML_ONLY: $mailer->setHtmlBody($this->_message->getStr('htmlContent') ?: null); break; case SendType::HTML_AND_PLAIN: $mailer->setTextBody($this->_message->getStr('plainText') ?: null); $mailer->setHtmlBody($this->_message->getStr('htmlContent') ?: null); break; } if ($mailer instanceof \Cubex\Email\Service\Mail) { $mailer->addHeader("X-Defero-MID", uniqid(class_shortname($mailer), true)); $mailer->addHeader("X-Defero-PID", getmypid()); } $campaignId = $this->_message->getStr('campaignId'); if ($mailer instanceof DatabaseMailer) { $mailer->setCampaignId($campaignId); } $mailer->setFrom($this->_message->getStr('senderEmail'), $this->_message->getStr('senderName')); $returnPath = $this->_message->getStr('returnPath'); if ($returnPath) { $mailer->setSender($returnPath); } $replyTo = $this->_message->getStr('replyTo'); if ($replyTo) { $mailer->setSender($replyTo); } else { $mailer->setSender($this->_message->getStr('senderEmail'), $this->_message->getStr('senderName')); } try { $result = $mailer->send(); } catch (\Exception $e) { Log::debug($e->getMessage()); $result = false; } $hour = time(); $hour -= $hour % 3600; if (isset($userData['statskey'])) { $brandStatsCf = MailStatisticsByBrand::cf(); $column = $hour . '|failed|' . $userData['statskey'] . '|' . $userData['language']; if ($result !== false) { $column = $hour . '|' . ($campaignActive ? 'sent' : 'test'); $column .= '|' . $userData['statskey'] . '|' . $userData['language']; $brandStatsCf->increment($campaignId, $column); } else { $brandStatsCf->increment($campaignId, $column); } } $statsCf = MailStatistic::cf(); $column = $hour . '|failed-' . $userData['language']; if ($result !== false) { $column = $hour . '|' . ($campaignActive ? 'sent' : 'test'); $column .= '-' . $userData['language']; $statsCf->increment($campaignId, $column); } else { $statsCf->increment($campaignId, $column); } if (!$userData['user_id']) { $userData['user_id'] = 0; } MailerLog::addLogEntry($userData['user_id'], $campaignId); if ($result !== false) { if ($campaignActive) { SentEmailLog::addLogEntry($userData['user_id'], $campaignId); } } return false; }