PHP KDGLoader示例

示例#1

文件： NewhtfCollectProductsCrawler.php 项目： typ6a/newhtf

<?php

KDGLoader::loadEntityClass('Product');
KDGLoader::loadModelClass('ProductModel');
KDGLoader::loadModelClass('CategoryModel');
KDGLoader::loadLibraryClass('parsers/NewhtfProductsListParser');
class NewhtfCollectProductsCrawler extends Crawler
{
    protected function crawl()
    {
        $categories = CategoryModel::findAllMain();
        foreach ($categories as $category) {
            $html = $this->requestProductsListPage($category->url);
            if ($html) {
                $this->parseProductsList($html, $category);
            }
        }
    }
    protected function requestProductsListPage($url)
    {
        $this->sendBuffered('category: ' . $url);
        return $this->makeRequest($url, false, false, false);
    }
    protected function parseProductsList($html, $category)
    {
        $parser = new NewhtfProductsListParser($html);
        foreach ($parser->products_list as $data) {
            $entity = ProductModel::findOneByUrl($data['url']);
            if (!$entity) {
                $entity = new Product();
            }

示例#2

文件： config.php 项目： typ6a/newhtf

set_time_limit(600);
require_once 'debug.php';
require_once 'const.php';
require_once BASE_DIR . 'classes/KDGLoader.php';
$config = array();
$config['db_user'] = DB_USERNAME;
$config['db_password'] = DB_PASSWORD;
$config['db_host'] = DB_SERVER;
$config['db_name'] = DB_DATABASE;
KDGLoader::loadLibraryClass('KDGGeneral');
KDGLoader::loadLibraryClass('KDGSession');
KDGLoader::loadLibraryClass('KDGModel');
KDGLoader::loadLibraryClass('Encoding');
KDGLoader::loadLibraryClass('KDGMemory');
KDGLoader::loadLibraryClass('ISDString');
KDGLoader::loadLibraryClass('Inflector');
KDGLoader::loadLibraryClass('KDGEntity');
KDGLoader::loadLibraryClass('KDGParser');
KDGLoader::loadLibraryClass('KDGCrawler');
KDGLoader::loadLibraryClass('KDGDatabase');
KDGLoader::loadLibraryClass('KDGInflector');
if (isConsole() && isset($argv) && $argv) {
    foreach ($argv as $val) {
        if (stristr($val, '=')) {
            $parts = explode('=', $val);
            if (isset($parts[0]) && isset($parts[1])) {
                setRequestParameter($parts[0], $parts[1]);
            }
        }
    }
}

示例#3

文件： NewhtfUpdateProductParser.php 项目： typ6a/newhtf

<?php

KDGLoader::loadEntityClass('Product');
KDGLoader::loadEntityClass('ProductProperty');
KDGLoader::loadEntityClass('ProductToProperty');
KDGLoader::loadModelClass('PropertyModel');
KDGLoader::loadEntityClass('ProductImage');
KDGLoader::loadModelClass('ProductImageModel');
class NewhtfUpdateProductParser extends KDGParser
{
    protected $product = null;
    public function __construct($response, $product)
    {
        $this->product = $product;
        parent::__construct($response);
    }
    protected function parseDataObject()
    {
    }
    protected function hasDataObject()
    {
        return true;
    }
    protected function parseTitle()
    {
        return $this->getResponseObject()->query('//h1')->item(0)->nodeValue;
    }
    protected function parsePrice()
    {
        $items = $this->getResponseObject()->query('//span[@id="ys_top_price"]/span[@class="allSumMain"]');
        if ($items->length > 0) {

示例#4

文件： KDGEntity.php 项目： typ6a/newhtf

 protected function createEntity($arr = array())
 {
     $class = ucfirst($this->getEntityName());
     if (!class_exists($class)) {
         KDGLoader::loadEntityClass($class);
     }
     if (class_exists($class)) {
         return new $class($arr);
     }
     return null;
 }

示例#5

文件： KDGService.php 项目： typ6a/newhtf

<?php

KDGLoader::loadLibraryClass('vendor/SimpleHtmlDom/simple_html_dom');
class KDGService
{
    protected $cacheLifetime = 86400;
    // 24 hours
    protected $data = array();
    protected $request = array();
    protected $rawLinks = array();
    protected $comparedData = array();
    const SERVICE_SEARCH_STATUS_TRUE = 1;
    const SERVICE_SEARCH_STATUS_FALSE = 0;
    const SERVICE_SEARCH_STATUS_ERROR = 2;
    protected $ServiceHost = null;
    protected $ServiceCode = null;
    protected $ServiceTitle = null;
    protected $ServiceMethod = 'html';
    protected $ServiceEnabled = true;
    protected $ServiceStatus = self::SERVICE_SEARCH_STATUS_FALSE;
    // not found anything by default
    protected $ServiceStatusErrorMessage = null;
    protected $company = null;
    protected $rawLink = null;
    protected $rawLinkDom = null;
    protected $rawLinksDom = null;
    protected $rawLinksHtml = null;
    protected $rawLinksLimit = 5;
    protected $rawLinkRequestAttempt = 1;
    protected $html_response = null;
    protected $html_response_object = null;

示例#6

文件： 1.collect_main_category.php 项目： typ6a/newhtf

<?php

require_once 'include/config.php';
set_time_limit(0);
ini_set('memory_limit', '256M');
KDGLoader::loadLibraryClass('crawlers/NewhtfCategoryCrawler');
$crawler = new NewhtfCategoryCrawler();
//$crawler->crawlMainCategories();
//$crawler->crawlMiddleCategories();
//$crawler->crawlSubCategories();
//$product_url = 'http://newhtf.ru/catalog/paneli-interernye-svetilniki/svetilniki_serii_kub/svetodiodnyy_svetilnik_dlya_interera_htf_cub_6_plt_6w_nw.html';
//KDGLoader::loadLibraryClass('parsers/NewhtfProductParser');
//$html = file_get_contents($product_url);
//new NewhtfProductParser($html);

示例#7

文件： ProductModel.php 项目： typ6a/newhtf

<?php

KDGLoader::loadLibraryClass('KDGModel');
class ProductModel extends KDGModel
{
    public static function getQuery()
    {
        return KDGDatabase::create()->entity('product')->from('product p');
    }
    public static function findAll()
    {
        return self::getQuery()->execute();
    }
    public static function findOneById($id)
    {
        return self::getQuery()->addWhere('p.id = ?', $id)->fetchOne();
    }
    public static function findOneByTitle($title)
    {
        return self::getQuery()->where('p.title = ?', $title)->fetchOne();
    }
    public static function findOneByUrl($url)
    {
        return self::getQuery()->debug(false)->where('p.url = ?', $url)->fetchOne();
    }
}

示例#8

文件： 3.update_products.php 项目： typ6a/newhtf

<?php

require_once 'include/config.php';
set_time_limit(0);
ini_set('memory_limit', '256M');
KDGLoader::loadLibraryClass('crawlers/NewhtfUpdateProductsCrawler');
new NewhtfUpdateProductsCrawler();
//$product_url = 'http://newhtf.ru/catalog/paneli-interernye-svetilniki/svetilniki_serii_kub/svetodiodnyy_svetilnik_dlya_interera_htf_cub_6_plt_6w_nw.html';
//KDGLoader::loadLibraryClass('parsers/NewhtfProductParser');
//$html = file_get_contents($product_url);
//new NewhtfProductParser($html);

示例#9

文件： NewhtfUpdateProductsCrawler.php 项目： typ6a/newhtf

<?php

KDGLoader::loadEntityClass('Product');
KDGLoader::loadModelClass('ProductModel');
KDGLoader::loadLibraryClass('parsers/NewhtfUpdateProductParser');
class NewhtfUpdateProductsCrawler extends Crawler
{
    protected function crawl()
    {
        $products = ProductModel::findAll();
        foreach ($products as $product) {
            sleep(0.2);
            $html = $this->requestProductPage($product->url);
            if ($html) {
                $this->parseProduct($html, $product);
            }
        }
    }
    protected function requestProductPage($url)
    {
        $this->sendBuffered('[product url]: ' . $url);
        return $this->makeRequest($url, false, false, false);
    }
    protected function parseProduct($html, $product)
    {
        new NewhtfUpdateProductParser($html, $product);
    }
}

示例#10

文件： CollectXbox360ContentCrawler.php 项目： typ6a/newhtf

<?php

KDGLoader::loadLibraryClass('parsers/Xbox360ContentPageParser');
class CollectXbox360ContentCrawler extends Crawler
{
    protected $page = 1;
    protected $pagemax = 22;
    public function execute()
    {
        while ($this->page <= $this->pagemax) {
            $this->collectContent();
            $this->page++;
        }
    }
    protected function collectContent()
    {
        $this->response = null;
        $this->requestPage();
        if ($this->response) {
            $this->parsePage();
        }
    }
    protected function strToHex($string)
    {
        $hex = '';
        for ($i = 0; $i < strlen($string); $i++) {
            $hex .= dechex(ord($string[$i]));
        }
        return $hex;
    }
    protected function requestPage()

示例#11

文件： KDGDatabase.php 项目： typ6a/newhtf

 public function createEntity($arr = array())
 {
     $class = KDGInflector::classify($this->getEntityName());
     if (!class_exists($class)) {
         KDGLoader::loadEntityClass($class);
     }
     if (class_exists($class)) {
         return new $class($arr);
     }
     return null;
 }

示例#12

文件： NewhtfCategoryCrawler.php 项目： typ6a/newhtf

<?php

KDGLoader::loadEntityClass('Category');
KDGLoader::loadModelClass('CategoryModel');
KDGLoader::loadLibraryClass('parsers/NewhtfMainCategoryParser');
KDGLoader::loadLibraryClass('parsers/NewhtfMiddleCategoryParser');
KDGLoader::loadLibraryClass('parsers/NewhtfSubCategoryParser');
class NewhtfCategoryCrawler extends Crawler
{
    protected $mainCategories = [];
    protected $middleCategories = [];
    protected $subCategories = [];
    protected function crawl()
    {
    }
    /* MAIN CATEGORIES */
    public function crawlMainCategories()
    {
        $this->response = null;
        $this->requestMainCategoriesPage();
        if ($this->response) {
            $this->parseMainCategories();
        }
    }
    protected function requestMainCategoriesPage()
    {
        $url = 'http://newhtf.ru/catalog/';
        getDiffTime(1);
        getDiffMemory(1);
        $this->sendBuffered($url);
        $this->response = $this->makeRequest($url, false, false, false);

示例#13

文件： 2.collect_products.php 项目： typ6a/newhtf

<?php

require_once 'include/config.php';
set_time_limit(0);
ini_set('memory_limit', '256M');
KDGLoader::loadLibraryClass('crawlers/NewhtfCollectProductsCrawler');
new NewhtfCollectProductsCrawler();
//$product_url = 'http://newhtf.ru/catalog/paneli-interernye-svetilniki/svetilniki_serii_kub/svetodiodnyy_svetilnik_dlya_interera_htf_cub_6_plt_6w_nw.html';
//KDGLoader::loadLibraryClass('parsers/NewhtfProductParser');
//$html = file_get_contents($product_url);
//new NewhtfProductParser($html);

示例#14

文件： XboxOneDLCContentDetailsPageParser.php 项目： typ6a/newhtf

KDGLoader::loadEntityClass('Developer');
KDGLoader::loadModelClass('DeveloperModel');
KDGLoader::loadEntityClass('Publisher');
KDGLoader::loadModelClass('PublisherModel');
KDGLoader::loadEntityClass('Content');
KDGLoader::loadModelClass('ContentModel');
KDGLoader::loadEntityClass('Genre');
KDGLoader::loadModelClass('GenreModel');
KDGLoader::loadEntityClass('ContentToGenre');
KDGLoader::loadModelClass('ContentToGenreModel');
KDGLoader::loadEntityClass('Feature');
KDGLoader::loadModelClass('FeatureModel');
KDGLoader::loadEntityClass('ContentToFeature');
KDGLoader::loadModelClass('ContentToFeatureModel');
KDGLoader::loadModelClass('CategoryModel');
KDGLoader::loadModelClass('ContentToCategoryModel');
class XboxOneDLCContentDetailsPageParser extends KDGParser
{
    protected $RawItem = null;
    protected $content_object = null;
    public function __construct($response, $content_object)
    {
        $this->content_object = $content_object;
        parent::__construct($response);
    }
    protected function hasDataObject()
    {
        return true;
    }
    protected function parseValue($val, $pattern)
    {