private function init()
{
if (!$this->gearmanTaskPool) {
$this->gearmanTaskPool = new GearmanTaskPool();
$this->gearmanTaskPool->setServers($this->servers);
$this->gearmanTaskPool->setMaxTasks($this->maxTasks);
$this->gearmanTaskPool->setTaskCallback([$this, '_onTask']);
$this->gearmanTaskPool->setFunctionName($this->functionName);
$this->gearmanTaskPool->setWorkloadCallback(function () {
$url = $this->repository->next();
return $url ? GearmanToolsUtils::packMessage($url) : false;
});
}
}
public function run()
{
while ($url = $this->repository->next()) {
try {
$this->processorPool->process($url);
} catch (\Exception $e) {
if ($this->logger) {
$this->logger->error("Failed to process url [{$url->getId()}] \"{$url->getUrl()}\": {$e->getMessage()}");
}
}
foreach ($this->processorPool->getExtractedUrls() as $eu) {
$this->repository->insert($eu);
}
$this->repository->done($url);
// log
$c = count($this->processorPool->getExtractedUrls());
$this->logger->info("url: {$url->getUrl()} / status: {$url->getStatus()} / extracted: {$c}");
}
}
