/** * Метод запускается по крон собирает данные по Летуаль * * @param $offset * @param $total * * @return int */ public function actionLetual($offset, $total) { $entity = new LetualLink(); do { $links = $entity->getLinks($offset, 20); if (!empty($links) && $offset < $total) { foreach ($links as $link) { \Yii::info(sprintf('Обработка: %s ', $link->link), 'cron'); $crawler = $this->getData($link->link); if (!$crawler) { \Yii::error(sprintf('Не удалось получить страницу: %s ', $link->link), 'cron'); continue; } $attributes = ['link' => $link->link, 'group' => $link->group, 'category' => $link->category, 'sub_category' => $link->sub_category]; $service = new ParserService(); $result = $service->productParse($crawler, ParserService::LET, $attributes); foreach ($result as $res) { if ($res instanceof Response) { if (empty($res->getPrice()) || empty($res->getTitle())) { $this->setDeleted($res, new LetualProduct()); \Yii::error(sprintf('Ошибка обработки: %s : цена или заголовок не найдены', $link->link), 'cron'); } else { $this->saveLetualResult($res); } } } unset($result); } $z = 1; $offset += 20; unset($links); unset($client); } else { $z = 0; } } while ($z > 0); (new LetualProduct())->setDeleted(); return 0; }
/** * @return int */ public function actionIndex() { $entity = new LetualLink(); $offset = 0; do { $links = $entity->getLinks($offset, 20); if (!empty($links)) { foreach ($links as $link) { $client = new Client(); $crawler = $client->request('GET', $link->link); $result = $crawler->filter('table.atg_store_productSummary tr')->each(function ($node) { $title = $node->filter('td.item h2')->each(function ($subNode) { return $subNode->text(); }); $article = $node->filter('td.item p.article')->each(function ($subNode) { return $subNode->text(); }); $description = $node->filter('td.item p.description')->each(function ($subNode) { return $subNode->text(); }); $price = $node->filter('td.price')->each(function ($subNode) { $oldPrice = $subNode->filter('p.old_price')->each(function ($subsNode) { return $subsNode->text(); }); $newPrice = $subNode->filter('p.new_price')->each(function ($subsNode) { return $subsNode->text(); }); $newPrice = trim(reset($newPrice)); $newPrice = str_replace(' ', '', $newPrice); $newPrice = str_replace('*', '', $newPrice); $newPrice = str_replace('\\r', '', $newPrice); $newPrice = str_replace('\\n', '', $newPrice); $oldPrice = trim(reset($oldPrice)); $oldPrice = str_replace(' ', '', $oldPrice); return ['oldPrice' => $oldPrice, 'newPrice' => $newPrice]; }); $image = $node->filter('td img')->each(function ($node) { return $this->url . $node->attr('src'); }); $article = trim(reset($article)); $article = str_replace('Артикул ', '', $article); $article = preg_replace("/[^a-zA-Z0-9]/", "", $article); return ['title' => reset($title), 'article' => $article, 'description' => reset($description), 'price' => reset($price), 'image' => reset($image)]; }); $brand = $crawler->filter('#brandImage')->each(function ($node) { return $node->attr('alt'); }); $result = $this->checkArray($result); //Если не нашли картинку в списке выбора if (empty($result['image'])) { $image = $crawler->filter('div.atg_store_productImage img')->each(function ($node) { return $this->url . $node->attr('src'); }); $result['image'] = reset($image); } $result['brand'] = reset($brand); $result['link'] = $link->link; $result['group'] = $link->group; $result['category'] = $link->category; $result['sub_category'] = $link->sub_category; $this->saveResult($result); } $z = 1; $offset += 20; unset($links); unset($client); } else { $z = 0; } } while ($z > 0); return 0; }