public function actionIndex() { /** @var $entity RivegaucheLink */ $entity = new RivegaucheLink(); $offset = 0; do { $links = $entity->getLinks($offset, 5); if (!empty($links)) { foreach ($links as $link) { \Yii::info(sprintf('Обрабатываем: %s ', $link['link']), 'cron'); $client = new Client(); $guzzle = $client->getClient(); $client->getClient()->setDefaultOption('config/curl/' . CURLOPT_PROXY, 'http://141.101.118.147:80'); //$client->getClient()->setDefaultOption('config/curl/'.CURLOPT_TIMEOUT, 10); $client->getClient()->setDefaultOption('config/curl/' . CURLOPT_CONNECTTIMEOUT, 10); $client->setClient($guzzle); /*$guzzle = $client->getClient(); $guzzle->setDefaultOption('timeout', 10); $client->getClient()->setDefaultOption('config/curl/'.CURLOPT_TIMEOUT_MS, 100); $client->getClient()->setDefaultOption('config/curl/'.CURLOPT_CONNECTTIMEOUT, 5); $client->getClient()->setDefaultOption('config/curl/'.CURLOPT_RETURNTRANSFER, true); $client->setClient($guzzle);*/ $crawler = $client->request('GET', $link['link']); \Yii::info(sprintf('Извлекаем тело: %s ', $link['link']), 'cron'); $head = $this->getHtml($crawler, true); \Yii::info(sprintf('HEAD тело: %s ', $link['link']), 'cron'); if (!empty($head['links'])) { foreach ($head['links'] as $l) { $crawler = $client->request('GET', $l); $subHead = $this->getHtml($crawler, false); $subHead['link'] = $l; $this->saveResult($subHead, $link); } } if (empty($head['title'])) { $head = $this->getPromoHTML($crawler, true); } if (empty($head['title'])) { $head = $this->getPromo2HTML($crawler, true); } $head['link'] = $link['link']; $this->saveResult($head, $link); unset($node); unset($subNode); unset($head); } $z = 1; $offset += 5; unset($links); unset($client); } else { $z = 0; } } while ($z > 0); return 0; }
/** * Метод запускается по крон собирает данные по РивГош * @param $offset * @param $total * * @return int */ public function actionRivegauche($offset, $total) { /** @var $entity RivegaucheLink */ $entity = new RivegaucheLink(); do { $links = $entity->getLinks($offset, 20); if (!empty($links) && $offset < $total) { foreach ($links as $link) { \Yii::info(sprintf('Обработка: %s ', $link->link), 'cron'); $crawler = $this->getData($link->link); if (!$crawler) { continue; } $attributes = ['link' => $link->link, 'group' => $link->group, 'category' => $link->category, 'sub_category' => $link->sub_category]; $service = new ParserService(); $result = $service->productParse($crawler, ParserService::RIV, $attributes); if (empty($result->getArticle()) || empty($result->getPrice()) || empty($result->getTitle())) { $this->setDeleted($result, new RivegaucheProduct()); $errorPr = ErrorProcessing::findOne(['link' => $link->link]); if (!$errorPr) { $errorPr = new ErrorProcessing(); } $errorPr->competitor = ErrorProcessing::RIVE; $errorPr->link = $link->link; $errorPr->processing = 0; $errorPr->comment = null; $errorPr->error = 'Ошибка обработки:'; if (empty($result->getArticle())) { $errorPr->error .= ' артикул'; } if (empty($result->getPrice())) { $errorPr->error .= ' цена'; } if (empty($result->getTitle())) { $errorPr->error .= ' заголовок'; } $errorPr->error .= ' не найдены'; $errorPr->save(); } else { $this->saveResult($result); } foreach ($result->getUrls() as $url) { $attributes = ['link' => $url, 'group' => $link->group, 'category' => $link->category, 'sub_category' => $link->sub_category]; $crawler = $this->getData($url); if ($crawler) { $service = new ParserService(); $result = $service->productParse($crawler, ParserService::RIV, $attributes); if (empty($result->getArticle()) || empty($result->getPrice()) || empty($result->getTitle())) { $this->setDeleted($result, new RivegaucheProduct()); $errorProcess = ErrorProcessing::findOne(['link' => $url]); if (!$errorProcess) { $errorProcess = new ErrorProcessing(); } $errorProcess->competitor = ErrorProcessing::RIVE; $errorProcess->link = $url; $errorProcess->processing = 0; $errorProcess->comment = null; $errorProcess->error = 'Ошибка обработки:'; if (empty($result->getArticle())) { $errorProcess->error .= ' артикул'; } if (empty($result->getPrice())) { $errorProcess->error .= ' цена'; } if (empty($result->getTitle())) { $errorProcess->error .= ' заголовок'; } $errorProcess->error .= ' не найдены'; $errorProcess->save(); } else { $this->saveResult($result); } } } unset($node); unset($service); unset($result); unset($head); } $z = 1; $offset += 20; unset($links); unset($client); } else { $z = 0; } } while ($z > 0); return 0; }