public function actionIndex()
{
/** @var $entity RivegaucheLink */
$entity = new RivegaucheLink();
$offset = 0;
do {
$links = $entity->getLinks($offset, 5);
if (!empty($links)) {
foreach ($links as $link) {
\Yii::info(sprintf('Обрабатываем: %s ', $link['link']), 'cron');
$client = new Client();
$guzzle = $client->getClient();
$client->getClient()->setDefaultOption('config/curl/' . CURLOPT_PROXY, 'http://141.101.118.147:80');
//$client->getClient()->setDefaultOption('config/curl/'.CURLOPT_TIMEOUT, 10);
$client->getClient()->setDefaultOption('config/curl/' . CURLOPT_CONNECTTIMEOUT, 10);
$client->setClient($guzzle);
/*$guzzle = $client->getClient();
$guzzle->setDefaultOption('timeout', 10);
$client->getClient()->setDefaultOption('config/curl/'.CURLOPT_TIMEOUT_MS, 100);
$client->getClient()->setDefaultOption('config/curl/'.CURLOPT_CONNECTTIMEOUT, 5);
$client->getClient()->setDefaultOption('config/curl/'.CURLOPT_RETURNTRANSFER, true);
$client->setClient($guzzle);*/
$crawler = $client->request('GET', $link['link']);
\Yii::info(sprintf('Извлекаем тело: %s ', $link['link']), 'cron');
$head = $this->getHtml($crawler, true);
\Yii::info(sprintf('HEAD тело: %s ', $link['link']), 'cron');
if (!empty($head['links'])) {
foreach ($head['links'] as $l) {
$crawler = $client->request('GET', $l);
$subHead = $this->getHtml($crawler, false);
$subHead['link'] = $l;
$this->saveResult($subHead, $link);
}
}
if (empty($head['title'])) {
$head = $this->getPromoHTML($crawler, true);
}
if (empty($head['title'])) {
$head = $this->getPromo2HTML($crawler, true);
}
$head['link'] = $link['link'];
$this->saveResult($head, $link);
unset($node);
unset($subNode);
unset($head);
}
$z = 1;
$offset += 5;
unset($links);
unset($client);
} else {
$z = 0;
}
} while ($z > 0);
return 0;
}
/**
* Метод запускается по крон собирает данные по РивГош
* @param $offset
* @param $total
*
* @return int
*/
public function actionRivegauche($offset, $total)
{
/** @var $entity RivegaucheLink */
$entity = new RivegaucheLink();
do {
$links = $entity->getLinks($offset, 20);
if (!empty($links) && $offset < $total) {
foreach ($links as $link) {
\Yii::info(sprintf('Обработка: %s ', $link->link), 'cron');
$crawler = $this->getData($link->link);
if (!$crawler) {
continue;
}
$attributes = ['link' => $link->link, 'group' => $link->group, 'category' => $link->category, 'sub_category' => $link->sub_category];
$service = new ParserService();
$result = $service->productParse($crawler, ParserService::RIV, $attributes);
if (empty($result->getArticle()) || empty($result->getPrice()) || empty($result->getTitle())) {
$this->setDeleted($result, new RivegaucheProduct());
$errorPr = ErrorProcessing::findOne(['link' => $link->link]);
if (!$errorPr) {
$errorPr = new ErrorProcessing();
}
$errorPr->competitor = ErrorProcessing::RIVE;
$errorPr->link = $link->link;
$errorPr->processing = 0;
$errorPr->comment = null;
$errorPr->error = 'Ошибка обработки:';
if (empty($result->getArticle())) {
$errorPr->error .= ' артикул';
}
if (empty($result->getPrice())) {
$errorPr->error .= ' цена';
}
if (empty($result->getTitle())) {
$errorPr->error .= ' заголовок';
}
$errorPr->error .= ' не найдены';
$errorPr->save();
} else {
$this->saveResult($result);
}
foreach ($result->getUrls() as $url) {
$attributes = ['link' => $url, 'group' => $link->group, 'category' => $link->category, 'sub_category' => $link->sub_category];
$crawler = $this->getData($url);
if ($crawler) {
$service = new ParserService();
$result = $service->productParse($crawler, ParserService::RIV, $attributes);
if (empty($result->getArticle()) || empty($result->getPrice()) || empty($result->getTitle())) {
$this->setDeleted($result, new RivegaucheProduct());
$errorProcess = ErrorProcessing::findOne(['link' => $url]);
if (!$errorProcess) {
$errorProcess = new ErrorProcessing();
}
$errorProcess->competitor = ErrorProcessing::RIVE;
$errorProcess->link = $url;
$errorProcess->processing = 0;
$errorProcess->comment = null;
$errorProcess->error = 'Ошибка обработки:';
if (empty($result->getArticle())) {
$errorProcess->error .= ' артикул';
}
if (empty($result->getPrice())) {
$errorProcess->error .= ' цена';
}
if (empty($result->getTitle())) {
$errorProcess->error .= ' заголовок';
}
$errorProcess->error .= ' не найдены';
$errorProcess->save();
} else {
$this->saveResult($result);
}
}
}
unset($node);
unset($service);
unset($result);
unset($head);
}
$z = 1;
$offset += 20;
unset($links);
unset($client);
} else {
$z = 0;
}
} while ($z > 0);
return 0;
}