public function runBatch()
{
$results = [];
/**
* @var $message IProcessMessage
*/
foreach ($this->_batch as $taskId => $message) {
$cid = $message->getInt('campaignId');
$started = $message->getInt('startedAt');
$lastSent = $message->getInt('lastSent', 0);
$startId = $message->getInt('startId', null);
$endId = $message->getInt('endId', 0);
$additionalData = $message->getRaw('additionalData', null);
$campaign = new Campaign($cid);
$dataSource = $campaign->getDataSource();
if ($startId == null) {
$startId = $dataSource->getLastId($cid, $taskId);
}
try {
$dataSource->process($taskId, $cid, $started, $lastSent, $startId, $endId, $additionalData);
$results[$taskId] = true;
} catch (\Exception $e) {
Log::error('Campaign ' . $cid . ': ' . $e->getMessage());
$results[$taskId] = false;
}
}
$this->_batch = [];
return $results;
}
public function execute()
{
$this->_logger->setInstanceName($this->instanceName);
$this->_pidFile = new PidFile("", $this->instanceName);
while (true) {
$startedAt = time();
$startedAt -= $startedAt % 60;
$collection = new RecordCollection(new Campaign());
if (!$collection->hasMappers()) {
Log::warning('No mappers found');
}
foreach ($collection as $campaign) {
/** @var Campaign $campaign */
if ($campaign->isDue($startedAt)) {
try {
Defero::pushCampaign($campaign->id(), $startedAt);
if (CronParser::isValid($campaign->sendAt)) {
// check average sends on scheduled
$avgEndDate = (new \DateTime())->setTimestamp($startedAt);
$avgStartDate = CronParser::prevRun($campaign->sendAt, $avgEndDate);
$avgEndDate->sub($avgStartDate->diff($avgEndDate));
$avgStartDate->setTime($avgStartDate->format('H') - 1, 0, 0);
$latestStats = MailStatistic::getCampaignStats($campaign->id(), $avgStartDate, $avgEndDate);
$diff = $avgStartDate->diff($avgEndDate);
$diffLatest = max(1, intval($diff->format('%i')) + intval($diff->format('%h') * 60) + intval($diff->format('%d') * 3600));
$latestHourly = $latestStats->sent / $diffLatest * 60;
if ($campaign->warnMin && $latestHourly < $campaign->warnMin || $campaign->warnMax && $latestHourly > $campaign->warnMax) {
Log::warning('Sending outside threshold', ['campaign' => $campaign->id(), 'average' => $latestHourly, 'warnMin' => $campaign->warnMin, 'warnMax' => $campaign->warnMax]);
}
}
} catch (\Exception $e) {
Log::error('Campaign ' . $campaign->id() . ': ' . $e->getMessage() . ' (Line: ' . $e->getLine() . ')');
}
} else {
Log::debug('Campaign ' . $campaign->id() . ' not due');
}
}
$endTime = time();
$endTime -= $endTime % 60;
if ($endTime == $startedAt) {
sleep(30);
}
}
}
public function runBatch()
{
$results = [];
/**
* @var $message IProcessMessage
*/
foreach ($this->_batch as $taskId => $message) {
try {
foreach ($message->getProcessQueue() as $currentProcess) {
if (!$this->runProcess($message, $currentProcess, $taskId)) {
break;
}
}
} catch (\Exception $e) {
Log::error($e->getMessage());
}
$results[$taskId] = true;
}
$this->_batch = [];
return $results;
}
public function processRange(BucketRange $range)
{
Log::info('Processing range ' . $range->id());
$this->_statsReporter->nextRange($range);
$this->_processor->resetRangeData();
$listOpts = ['Bucket' => $this->_bucketName, 'MaxKeys' => $this->_batchSize, 'Prefix' => $range->prefix];
try {
if ($this->_processor->shouldSaveProgress() && !empty($range->lastObject)) {
$lastObject = $range->lastObject;
Log::info('Continuing from key ' . $lastObject);
} else {
$lastObject = "";
}
$gcs = $this->_getGCS();
$finished = false;
while (!$finished) {
if ($lastObject != "") {
$listOpts['Marker'] = $lastObject;
}
EventManager::trigger(Events::LIST_BUCKET_START);
$objects = $gcs->listObjects($listOpts);
EventManager::trigger(Events::LIST_BUCKET_END);
$numObjects = count($objects['Contents']);
if ($numObjects > 0) {
$items = [];
foreach ($objects['Contents'] as $object) {
$items[] = new ObjectInfo($object['Key'], $object['Size'], $object['ETag'], $object['LastModified']);
}
$batchProcessed = $this->_processor->processBatch($items);
$lastObjectObj = end($items);
$lastObject = $lastObjectObj->objectKey;
$range->lastObject = $lastObject;
$this->_statsReporter->addItems($numObjects, $batchProcessed);
if ($this->_processor->shouldSaveProgress()) {
$range->rangeData = $this->_processor->getRangeData();
$range->saveChanges();
}
} else {
Log::info('Finished processing range ' . $range->prefix . ' (' . $this->_statsReporter->rangeTotalItems . ' total items)');
$finished = true;
}
$this->_statsReporter->displayReport();
}
RangeManager::rangeProcessed($range, $this->_statsReporter->rangeStartTime, $this->_processor->getRangeData());
} catch (\Exception $e) {
$msg = "Code " . $e->getCode();
$exMsg = $e->getMessage();
if ($exMsg != "") {
$msg .= ": " . $exMsg;
}
Log::error('Error processing range: ' . $msg . "\n\nBacktrace:\n" . $e->getTraceAsString());
if ($this->_processor->isFatalException($e)) {
RangeManager::rangeFailed($range, $this->_statsReporter->rangeStartTime, $msg);
} else {
RangeManager::requeueRange($range, $this->_statsReporter->rangeStartTime, $msg);
}
}
}
public function processRange(TokenRange $range)
{
Log::info("Processing range ID " . $range->id() . " from '" . $range->firstKey . "' to '" . $range->lastKey . "'...");
$this->_batchSizeTuner->reset();
$totalItems = 0;
$processedItems = 0;
$errors = 0;
$rangeStartTime = microtime(true);
$this->_processor->resetRangeData();
try {
$cf = $this->_getCF();
$cols = $this->_processor->requiredColumns();
if (is_numeric($cols)) {
$cols = new SlicePredicate(['slice_range' => $cf->makeSlice('', '', false, $cols)]);
}
$lastKey = $range->firstKey ? $range->firstKey : "";
$rangeLastKey = $range->lastKey ? $range->lastKey : "";
$finished = false;
while (!$finished) {
$this->_batchSizeTuner->nextBatch();
$batchSize = $this->_batchSizeTuner->getBatchSize();
$items = $this->_getKeysWithRetry($cf, $lastKey, $rangeLastKey, $batchSize, $cols);
//$items = $cf->getKeys($lastKey, $rangeLastKey, $batchSize, $cols);
if (!$items) {
Log::info("Found no more items in range");
break;
}
// Skip the last item in the range because this will be the
// first item in the next range
if ($rangeLastKey != "" && last_key($items) == $rangeLastKey) {
array_pop($items);
}
if ($this->_processor->supportsBatchProcessing()) {
try {
EventManager::trigger(Events::PROCESS_BATCH_START);
$batchProcessed = $this->_processor->processBatch($items);
EventManager::trigger(Events::PROCESS_BATCH_END);
$processedItems += $batchProcessed;
$totalItems += count($items);
$this->_statsReporter->processedItems += $batchProcessed;
$this->_statsReporter->totalItems += count($items);
} catch (BatchException $e) {
$errors += $e->getErrorCount();
$this->_statsReporter->errors += $e->getErrorCount();
$msg = $e->getMessage();
if ($msg != "") {
$msg = 'Error processing batch: ' . $msg;
}
Log::error($msg);
if ($this->_processor->stopOnErrors()) {
die;
}
}
} else {
foreach ($items as $key => $itemData) {
try {
if ($this->_processor->processItem($key, $itemData)) {
$processedItems++;
$this->_statsReporter->processedItems++;
}
} catch (ItemException $e) {
$errors++;
$this->_statsReporter->errors++;
Log::error('Error processing item ' . $key . ' : ' . $e->getMessage());
if ($this->_processor->stopOnErrors()) {
die;
}
}
$totalItems++;
$this->_statsReporter->totalItems++;
}
}
$lastKey = last_key($items);
if ($this->_processor->shouldSaveProgress()) {
$this->_scriptProgress->save($range->firstKey, $lastKey);
}
if ($lastKey == $rangeLastKey || count($items) < $batchSize) {
$finished = true;
}
$this->_statsReporter->displayReport($finished, $range, $totalItems, $processedItems, $errors, $rangeStartTime, $lastKey);
if ($errors > 0) {
break;
}
}
$rangeData = $this->_processor->getRangeData();
$range->rangeData = $rangeData ? json_encode($rangeData) : '';
$range->failed = 0;
$range->error = "";
} catch (\Exception $e) {
$msg = "Code " . $e->getCode();
$exMsg = $e->getMessage();
if ($exMsg != "") {
$msg .= ": " . $exMsg;
}
Log::error('Error processing range: ' . $msg . "\n\nBacktrace:\n" . $e->getTraceAsString());
$range->error = $msg;
// Check for non-fatal errors (i.e. Cassandra timeouts)
if ($e instanceof CassandraException && ($e->getCode() == 408 || $e->getCode() == 503 || starts_with($e->getMessage(), 'TSocket: timed out'))) {
$errors++;
} else {
$range->failed = 1;
}
}
if ($errors > 0) {
$this->_requeueRange($range);
} else {
$range->processing = 0;
$range->processed = 1;
$range->processingTime = microtime(true) - $rangeStartTime;
$range->totalItems = $totalItems;
$range->processedItems = $processedItems;
$range->errorCount = $errors;
$range->saveChanges();
}
}
