<?php
KDGLoader::loadEntityClass('Product');
KDGLoader::loadModelClass('ProductModel');
KDGLoader::loadModelClass('CategoryModel');
KDGLoader::loadLibraryClass('parsers/NewhtfProductsListParser');
class NewhtfCollectProductsCrawler extends Crawler
{
protected function crawl()
{
$categories = CategoryModel::findAllMain();
foreach ($categories as $category) {
$html = $this->requestProductsListPage($category->url);
if ($html) {
$this->parseProductsList($html, $category);
}
}
}
protected function requestProductsListPage($url)
{
$this->sendBuffered('category: ' . $url);
return $this->makeRequest($url, false, false, false);
}
protected function parseProductsList($html, $category)
{
$parser = new NewhtfProductsListParser($html);
foreach ($parser->products_list as $data) {
$entity = ProductModel::findOneByUrl($data['url']);
if (!$entity) {
$entity = new Product();
}
<?php
KDGLoader::loadEntityClass('Product');
KDGLoader::loadModelClass('ProductModel');
KDGLoader::loadLibraryClass('parsers/NewhtfUpdateProductParser');
class NewhtfUpdateProductsCrawler extends Crawler
{
protected function crawl()
{
$products = ProductModel::findAll();
foreach ($products as $product) {
sleep(0.2);
$html = $this->requestProductPage($product->url);
if ($html) {
$this->parseProduct($html, $product);
}
}
}
protected function requestProductPage($url)
{
$this->sendBuffered('[product url]: ' . $url);
return $this->makeRequest($url, false, false, false);
}
protected function parseProduct($html, $product)
{
new NewhtfUpdateProductParser($html, $product);
}
}
<?php
KDGLoader::loadEntityClass('Product');
KDGLoader::loadEntityClass('ProductProperty');
KDGLoader::loadEntityClass('ProductToProperty');
KDGLoader::loadModelClass('PropertyModel');
KDGLoader::loadEntityClass('ProductImage');
KDGLoader::loadModelClass('ProductImageModel');
class NewhtfUpdateProductParser extends KDGParser
{
protected $product = null;
public function __construct($response, $product)
{
$this->product = $product;
parent::__construct($response);
}
protected function parseDataObject()
{
}
protected function hasDataObject()
{
return true;
}
protected function parseTitle()
{
return $this->getResponseObject()->query('//h1')->item(0)->nodeValue;
}
protected function parsePrice()
{
$items = $this->getResponseObject()->query('//span[@id="ys_top_price"]/span[@class="allSumMain"]');
if ($items->length > 0) {
KDGLoader::loadEntityClass('Developer');
KDGLoader::loadModelClass('DeveloperModel');
KDGLoader::loadEntityClass('Publisher');
KDGLoader::loadModelClass('PublisherModel');
KDGLoader::loadEntityClass('Content');
KDGLoader::loadModelClass('ContentModel');
KDGLoader::loadEntityClass('Genre');
KDGLoader::loadModelClass('GenreModel');
KDGLoader::loadEntityClass('ContentToGenre');
KDGLoader::loadModelClass('ContentToGenreModel');
KDGLoader::loadEntityClass('Feature');
KDGLoader::loadModelClass('FeatureModel');
KDGLoader::loadEntityClass('ContentToFeature');
KDGLoader::loadModelClass('ContentToFeatureModel');
KDGLoader::loadModelClass('CategoryModel');
KDGLoader::loadModelClass('ContentToCategoryModel');
class XboxOneDLCContentDetailsPageParser extends KDGParser
{
protected $RawItem = null;
protected $content_object = null;
public function __construct($response, $content_object)
{
$this->content_object = $content_object;
parent::__construct($response);
}
protected function hasDataObject()
{
return true;
}
protected function parseValue($val, $pattern)
{
