/**
* Craw content from start url to the last url which got
* @param string $start_uri: The main start url to check
* @param string $md5sum_to_check: The last md5 checksum of url which is have been got
*
*/
public static function _Do_Craw_Content($start_uri, $md5sum_to_check)
{
self::_print_to_console('Begin fetch URI:<br/>' . $start_uri);
$can_next = true;
$next_link = '';
$backet = array();
$content = Vendor_Crawler::get_content_from_uri_by_curl($start_uri);
if ($content) {
self::_print_to_console('Get content done!');
// Create a DOM object
require_once Kohana::find_file('classes', 'vendor/simple_html_dom');
$html = new simple_html_dom();
// Load HTML from a string
$html->load($content['output']);
//begin find content
$container = $html->find('div[id="TopicList"]', 0);
if ($container) {
//begin get story
$stories = $container->find('div[class="cont09"]');
if (count($stories) > 0) {
//begin parse each story
$index = 0;
foreach ($stories as $story) {
if ($can_next) {
# find title,image,summary => send to $backet
$ok = true;
$str = array();
$img = $story->find('img', 0);
if ($img) {
$str['image'] = $img->src;
} else {
$ok = false;
}
unset($img);
$lnk = $story->find('a[class="lnk02"]', 0);
if ($lnk) {
$str['md5sum'] = md5($lnk->href);
if ($str['md5sum'] != $md5sum_to_check) {
$str['link'] = $lnk->href;
$str['title'] = trim($lnk->plaintext);
# $str['cung_1'] = $matches[];
$str['slug'] = Vendor_Crawler::toAscii($str['title']);
$pattern = '/([a-z]+[-]+[a-z]+)[-]va[-]([a-z]+[-]+[a-z]+)[-].*/ismUx';
preg_match($pattern, $str['slug'], $matches);
if (isset($matches[1]) && isset($matches[2])) {
$str['cung_1'] = $matches[1];
$str['cung_2'] = $matches[2];
} else {
$pattern = '/hai[-]nguoi[-]cung[-]+([a-z]+[-]+[a-z]+)[-].*/ismUx';
preg_match($pattern, $str['slug'], $matches);
if (isset($matches[1])) {
$str['cung_1'] = $str['cung_2'] = $matches[1];
} else {
$str['cung_1'] = $str['cung_2'] = '-';
}
}
} else {
$can_next = false;
}
# $str['cung_1'] = $matches;
} else {
$ok = false;
}
unset($lnk);
if ($can_next) {
$summary = $story->find('p[class="cont09txt"]', 0);
if ($summary) {
$str['summary'] = trim($summary->plaintext);
} else {
$ok = false;
}
}
if ($ok && $can_next) {
$str['date_check'] = date("Y-m-d h:i:s");
$backet[$index] = $str;
unset($str);
$index++;
}
} else {
return;
}
}
//EOF foreach
# get next page link
if ($can_next) {
$tmp = explode('?p=', $start_uri);
if (isset($tmp[1])) {
$next_link = $tmp[0] . "?p=" . (intval($tmp[1]) + 1);
} else {
# next page is the second
$next_link = $tmp[0] . "?p=2";
}
//echo $next_link;
}
//var_dump($backet);
//insert backet into db
if (count($backet > 0)) {
$succ = 0;
$failure = 0;
$dupp = 0;
foreach ($backet as $bag) {
if (Model_Horoscope_XungHopLinkBLL::CheckRecordByMd5Sum($bag['md5sum'])) {
$dupp++;
} else {
$item = new Model_Horoscope_XungHopLink();
$item->uri = $bag['link'];
$item->md5sum = $bag['md5sum'];
$item->title = $bag['title'];
$item->hinh_anh = $bag['image'];
$item->ngay_check = $bag['date_check'];
$item->summary = $bag['summary'];
$item->cung_1 = $bag['cung_1'];
$item->cung_2 = $bag['cung_2'];
$item->hoan_thanh = false;
$item->save();
if ($item->identifier()) {
$succ++;
} else {
$failure++;
}
}
}
}
self::_print_to_console('--> INSERT: ' . $succ . ' (ok) , ' . $failure . ' (false) , ' . $dupp . ' (duplicate) <-- <br/>');
flush();
} else {
self::_print_to_console('--> No story found!');
$can_next = false;
}
} else {
self::_print_to_console('Cant get main container!');
}
$html->clear();
unset($html);
unset($content);
//sleep a while
usleep(2500);
//do recursive
if ($can_next && $next_link != '') {
self::_Do_Craw_Content($next_link, $md5sum_to_check);
}
} else {
self::_print_to_console('Cant get content for:<br/>' . $start_uri);
}
}
