Ejemplo n.º 1
0
 public function run(\GearmanJob $job)
 {
     $url = GearmanToolsUtils::unpackMessage($job->workload());
     $error = 0;
     try {
         $this->processorPool->process($url);
         $extractedUrlsCount = count($this->processorPool->getExtractedUrls());
         $message = "{$url->getUrl()} / depth: {$url->getDepth()} / status: {$url->getStatus()} / extracted: {$extractedUrlsCount}";
     } catch (\Exception $e) {
         $message = "Failed to process url [{$url->getId()}] \"{$url->getUrl()}\": {$e->getMessage()}";
         $error = $e->getCode();
         $error = $error === 0 ? -1 : $error;
     }
     $result = ['url' => $url, 'extractedUrls' => $this->processorPool->getExtractedUrls(), 'error' => $error, 'message' => $message];
     $this->logger->info($message);
     $result = GearmanToolsUtils::packMessage($result);
     return $result;
 }
Ejemplo n.º 2
0
 public function run()
 {
     while ($url = $this->repository->next()) {
         try {
             $this->processorPool->process($url);
         } catch (\Exception $e) {
             if ($this->logger) {
                 $this->logger->error("Failed to process url [{$url->getId()}] \"{$url->getUrl()}\": {$e->getMessage()}");
             }
         }
         foreach ($this->processorPool->getExtractedUrls() as $eu) {
             $this->repository->insert($eu);
         }
         $this->repository->done($url);
         // log
         $c = count($this->processorPool->getExtractedUrls());
         $this->logger->info("url: {$url->getUrl()} / status: {$url->getStatus()} / extracted: {$c}");
     }
 }