public function runBatch() { $results = []; /** * @var $message IProcessMessage */ foreach ($this->_batch as $taskId => $message) { $cid = $message->getInt('campaignId'); $started = $message->getInt('startedAt'); $lastSent = $message->getInt('lastSent', 0); $startId = $message->getInt('startId', null); $endId = $message->getInt('endId', 0); $additionalData = $message->getRaw('additionalData', null); $campaign = new Campaign($cid); $dataSource = $campaign->getDataSource(); if ($startId == null) { $startId = $dataSource->getLastId($cid, $taskId); } try { $dataSource->process($taskId, $cid, $started, $lastSent, $startId, $endId, $additionalData); $results[$taskId] = true; } catch (\Exception $e) { Log::error('Campaign ' . $cid . ': ' . $e->getMessage()); $results[$taskId] = false; } } $this->_batch = []; return $results; }
public function execute() { $this->_logger->setInstanceName($this->instanceName); $this->_pidFile = new PidFile("", $this->instanceName); while (true) { $startedAt = time(); $startedAt -= $startedAt % 60; $collection = new RecordCollection(new Campaign()); if (!$collection->hasMappers()) { Log::warning('No mappers found'); } foreach ($collection as $campaign) { /** @var Campaign $campaign */ if ($campaign->isDue($startedAt)) { try { Defero::pushCampaign($campaign->id(), $startedAt); if (CronParser::isValid($campaign->sendAt)) { // check average sends on scheduled $avgEndDate = (new \DateTime())->setTimestamp($startedAt); $avgStartDate = CronParser::prevRun($campaign->sendAt, $avgEndDate); $avgEndDate->sub($avgStartDate->diff($avgEndDate)); $avgStartDate->setTime($avgStartDate->format('H') - 1, 0, 0); $latestStats = MailStatistic::getCampaignStats($campaign->id(), $avgStartDate, $avgEndDate); $diff = $avgStartDate->diff($avgEndDate); $diffLatest = max(1, intval($diff->format('%i')) + intval($diff->format('%h') * 60) + intval($diff->format('%d') * 3600)); $latestHourly = $latestStats->sent / $diffLatest * 60; if ($campaign->warnMin && $latestHourly < $campaign->warnMin || $campaign->warnMax && $latestHourly > $campaign->warnMax) { Log::warning('Sending outside threshold', ['campaign' => $campaign->id(), 'average' => $latestHourly, 'warnMin' => $campaign->warnMin, 'warnMax' => $campaign->warnMax]); } } } catch (\Exception $e) { Log::error('Campaign ' . $campaign->id() . ': ' . $e->getMessage() . ' (Line: ' . $e->getLine() . ')'); } } else { Log::debug('Campaign ' . $campaign->id() . ' not due'); } } $endTime = time(); $endTime -= $endTime % 60; if ($endTime == $startedAt) { sleep(30); } } }
public function runBatch() { $results = []; /** * @var $message IProcessMessage */ foreach ($this->_batch as $taskId => $message) { try { foreach ($message->getProcessQueue() as $currentProcess) { if (!$this->runProcess($message, $currentProcess, $taskId)) { break; } } } catch (\Exception $e) { Log::error($e->getMessage()); } $results[$taskId] = true; } $this->_batch = []; return $results; }
public function processRange(BucketRange $range) { Log::info('Processing range ' . $range->id()); $this->_statsReporter->nextRange($range); $this->_processor->resetRangeData(); $listOpts = ['Bucket' => $this->_bucketName, 'MaxKeys' => $this->_batchSize, 'Prefix' => $range->prefix]; try { if ($this->_processor->shouldSaveProgress() && !empty($range->lastObject)) { $lastObject = $range->lastObject; Log::info('Continuing from key ' . $lastObject); } else { $lastObject = ""; } $gcs = $this->_getGCS(); $finished = false; while (!$finished) { if ($lastObject != "") { $listOpts['Marker'] = $lastObject; } EventManager::trigger(Events::LIST_BUCKET_START); $objects = $gcs->listObjects($listOpts); EventManager::trigger(Events::LIST_BUCKET_END); $numObjects = count($objects['Contents']); if ($numObjects > 0) { $items = []; foreach ($objects['Contents'] as $object) { $items[] = new ObjectInfo($object['Key'], $object['Size'], $object['ETag'], $object['LastModified']); } $batchProcessed = $this->_processor->processBatch($items); $lastObjectObj = end($items); $lastObject = $lastObjectObj->objectKey; $range->lastObject = $lastObject; $this->_statsReporter->addItems($numObjects, $batchProcessed); if ($this->_processor->shouldSaveProgress()) { $range->rangeData = $this->_processor->getRangeData(); $range->saveChanges(); } } else { Log::info('Finished processing range ' . $range->prefix . ' (' . $this->_statsReporter->rangeTotalItems . ' total items)'); $finished = true; } $this->_statsReporter->displayReport(); } RangeManager::rangeProcessed($range, $this->_statsReporter->rangeStartTime, $this->_processor->getRangeData()); } catch (\Exception $e) { $msg = "Code " . $e->getCode(); $exMsg = $e->getMessage(); if ($exMsg != "") { $msg .= ": " . $exMsg; } Log::error('Error processing range: ' . $msg . "\n\nBacktrace:\n" . $e->getTraceAsString()); if ($this->_processor->isFatalException($e)) { RangeManager::rangeFailed($range, $this->_statsReporter->rangeStartTime, $msg); } else { RangeManager::requeueRange($range, $this->_statsReporter->rangeStartTime, $msg); } } }
public function processRange(TokenRange $range) { Log::info("Processing range ID " . $range->id() . " from '" . $range->firstKey . "' to '" . $range->lastKey . "'..."); $this->_batchSizeTuner->reset(); $totalItems = 0; $processedItems = 0; $errors = 0; $rangeStartTime = microtime(true); $this->_processor->resetRangeData(); try { $cf = $this->_getCF(); $cols = $this->_processor->requiredColumns(); if (is_numeric($cols)) { $cols = new SlicePredicate(['slice_range' => $cf->makeSlice('', '', false, $cols)]); } $lastKey = $range->firstKey ? $range->firstKey : ""; $rangeLastKey = $range->lastKey ? $range->lastKey : ""; $finished = false; while (!$finished) { $this->_batchSizeTuner->nextBatch(); $batchSize = $this->_batchSizeTuner->getBatchSize(); $items = $this->_getKeysWithRetry($cf, $lastKey, $rangeLastKey, $batchSize, $cols); //$items = $cf->getKeys($lastKey, $rangeLastKey, $batchSize, $cols); if (!$items) { Log::info("Found no more items in range"); break; } // Skip the last item in the range because this will be the // first item in the next range if ($rangeLastKey != "" && last_key($items) == $rangeLastKey) { array_pop($items); } if ($this->_processor->supportsBatchProcessing()) { try { EventManager::trigger(Events::PROCESS_BATCH_START); $batchProcessed = $this->_processor->processBatch($items); EventManager::trigger(Events::PROCESS_BATCH_END); $processedItems += $batchProcessed; $totalItems += count($items); $this->_statsReporter->processedItems += $batchProcessed; $this->_statsReporter->totalItems += count($items); } catch (BatchException $e) { $errors += $e->getErrorCount(); $this->_statsReporter->errors += $e->getErrorCount(); $msg = $e->getMessage(); if ($msg != "") { $msg = 'Error processing batch: ' . $msg; } Log::error($msg); if ($this->_processor->stopOnErrors()) { die; } } } else { foreach ($items as $key => $itemData) { try { if ($this->_processor->processItem($key, $itemData)) { $processedItems++; $this->_statsReporter->processedItems++; } } catch (ItemException $e) { $errors++; $this->_statsReporter->errors++; Log::error('Error processing item ' . $key . ' : ' . $e->getMessage()); if ($this->_processor->stopOnErrors()) { die; } } $totalItems++; $this->_statsReporter->totalItems++; } } $lastKey = last_key($items); if ($this->_processor->shouldSaveProgress()) { $this->_scriptProgress->save($range->firstKey, $lastKey); } if ($lastKey == $rangeLastKey || count($items) < $batchSize) { $finished = true; } $this->_statsReporter->displayReport($finished, $range, $totalItems, $processedItems, $errors, $rangeStartTime, $lastKey); if ($errors > 0) { break; } } $rangeData = $this->_processor->getRangeData(); $range->rangeData = $rangeData ? json_encode($rangeData) : ''; $range->failed = 0; $range->error = ""; } catch (\Exception $e) { $msg = "Code " . $e->getCode(); $exMsg = $e->getMessage(); if ($exMsg != "") { $msg .= ": " . $exMsg; } Log::error('Error processing range: ' . $msg . "\n\nBacktrace:\n" . $e->getTraceAsString()); $range->error = $msg; // Check for non-fatal errors (i.e. Cassandra timeouts) if ($e instanceof CassandraException && ($e->getCode() == 408 || $e->getCode() == 503 || starts_with($e->getMessage(), 'TSocket: timed out'))) { $errors++; } else { $range->failed = 1; } } if ($errors > 0) { $this->_requeueRange($range); } else { $range->processing = 0; $range->processed = 1; $range->processingTime = microtime(true) - $rangeStartTime; $range->totalItems = $totalItems; $range->processedItems = $processedItems; $range->errorCount = $errors; $range->saveChanges(); } }