private function do_get_temp()
{
$five_rows = Model_Horoscope_XungHopLinkBLL::GetRecords_UnProccessed($num_of_rows = 10);
if ($five_rows) {
$count = count($five_rows);
for ($i = 0; $i < $count; $i++) {
$_ID = $five_rows[$i]['id'];
$_URI = $five_rows[$i]['uri'];
$_Cung1 = $five_rows[$i]['cung_1'];
$_Cung2 = $five_rows[$i]['cung_2'];
$_Summary = $five_rows[$i]['summary'];
$_ImageLink = $five_rows[$i]['hinh_anh'];
//check uri in story table
if (Model_Horoscope_XungHopBLL::CheckRecordByURI($_URI)) {
//(dup) existed => not need to insert => update from tmp table
Model_Horoscope_XungHopLinkBLL::MarkAsDone($_ID, $_URI);
} else {
//begin get content
$content = Vendor_Crawler::get_content_from_uri_by_curl($_URI);
if ($content) {
//process content just got
Model_Horoscope_XungHopBLL::_Process_Recieved_Content($content['output'], $_Cung1, $_Cung2, $_Summary, $_URI, $_ID, $_ImageLink);
} else {
Model_Horoscope_XungHopBLL::_print_to_console(__("<--- CANT GET CONTENT --->"));
}
}
}
//EOF for loop
} else {
unset($five_rows);
return false;
}
unset($five_rows);
//do recursive
$this->do_get_temp();
}
public static function _Process_Recieved_Content($_HTML_CONTENT, $_Cung1, $_Cung2, $_Summary, $_SourceUri, $_LinkId, $_ImageLink)
{
if ($_HTML_CONTENT != '') {
// Create a DOM object
require_once Kohana::find_file('classes', 'vendor/simple_html_dom');
$html = new simple_html_dom();
// Load HTML from a string
$html->load($_HTML_CONTENT);
unset($_HTML_CONTENT);
if ($html) {
$story = new Model_Horoscope_XungHop();
$ktra = true;
if ($_Cung1 == '-' || $_Cung2 == '-') {
$ktra = FALSE;
}
$story->cung_1 = $_Cung1;
$story->cung_2 = $_Cung2;
$story->alias = $_Cung1 . '_' . $_Cung2;
if (self::CheckRecordByAlias($story->alias)) {
$story->alias = $_Cung1 . '__' . $_Cung2;
}
$story->tom_tat = $_Summary;
$story->ngay_tao = date("Y-m-d");
$story->url_nguon = $_SourceUri;
$story->auto_get = true;
//begin find elements
#find date post
$date = $html->find('div[class="datetime"]', 0);
if ($date) {
$d = explode(',', $date->plaintext);
if (isset($d[1])) {
//var_dump($d);
//exit;
$d1 = explode(' ', trim($d[1]));
list($ngay, $thang, $nam) = explode('/', $d1[0]);
$story->source_date = date("Y-m-d h:i:s", strtotime($nam . '-' . $thang . '-' . $ngay . ' ' . $d1[1] . ':00'));
} else {
$story->source_date = date("Y-m-d h:i:s");
}
} else {
$story->source_date = date("Y-m-d h:i:s");
}
//find content
$content = $html->find('div[id="content_document"]', 0);
if ($content) {
$string = $content->innertext;
# remove white space
$string = str_replace(array("\r\n", "\r", "\n", "\t"), '', $string);
$string = preg_replace('/(<!--.+?-->)/s', '', $string);
$string = preg_replace('@<a[^>]*>(.*)</a>@ismUx', '$1', $string);
$string = preg_replace('/<p[ ]class="pAuthor">.*<\\/p>/ismxU', '', $string);
$string = preg_replace('/<p[ ]class="pSource">.*<\\/p>/ismxU', '', $string);
$story->noi_dung = $string;
$story->kiem_tra = $ktra;
$story->save();
if ($story->identifier()) {
if ($ktra) {
//get image thumb => save to disk => update record in db
$path = 'assets/horoscope/xung-hop/' . $story->alias . '/';
$img = Vendor_Crawler::get_file_from_url_by_curl($_ImageLink, $save_to_path = $path, $file_name_to_set = $story->alias . '-thumb');
if ($img) {
//check file size, if = 0 -> mean file can't get
if (filesize($img) == 0) {
@copy('assets/horoscope/thumb_140.jpg', $img);
}
$story->hinh_anh = '/' . $img;
} else {
$story->hinh_anh = $_ImageLink;
}
} else {
$story->hinh_anh = $_ImageLink;
}
if ($ktra != FALSE) {
//print_r($img);
$html2 = new simple_html_dom();
$html2->load($story->noi_dung);
$images = $html2->find('img');
if (count($images) > 0) {
for ($i = 0; $i < count($images); $i++) {
unset($images[$i]->onclick);
$file_name = 'anh_' . $i + 1;
$get_file = Vendor_Crawler::get_file_from_url_by_curl($images[$i]->src, $save_to_path = $path, $file_name_to_set = $file_name);
if (filesize(ltrim($get_file, '/')) == 0) {
unset($images[$i]);
} else {
$images[$i]->src = '/' . $get_file;
}
}
}
$story->noi_dung = $html2->save();
$html2->clear();
unset($html2);
} else {
$story->hinh_anh = $_ImageLink;
}
$story->save();
//insert done => update from tmp table
Model_Horoscope_XungHopLinkBLL::UpdateRecordStatus($_LinkId);
self::_print_to_console('Done: ' . $_SourceUri);
} else {
self::_print_to_console('Fail:' . $_SourceUri);
}
} else {
self::_print_to_console('-> content not found');
return false;
}
$html->clear();
unset($html);
} else {
self::_print_to_console('-> cant load DOM obj');
return false;
}
} else {
self::_print_to_console('-> nothing to do');
return false;
}
}
private function _do_craw($_DOMAIN, $uri, $uri_to_check)
{
$can_next = true;
$next_link = '';
if ($uri == $uri_to_check) {
$can_next = false;
}
$content = Vendor_Crawler::get_content_from_uri_by_curl($uri);
if ($content && $can_next) {
$this->_print_to_console('Done get content!');
// Create a DOM object
require_once Kohana::find_file('classes', 'vendor/simple_html_dom');
$html = new simple_html_dom();
// Load HTML from a string
$html->load($content['output']);
$basket = array();
//begin find content
$story_1 = $html->find('div[id="nbm"]', 0);
if ($story_1) {
echo "FOUND STORY 1!" . PHP_EOL;
$lnk = $story_1->find('a', 0);
$img = $story_1->find('img', 0);
if ($lnk) {
if ($_DOMAIN . trim($lnk->href) != $uri_to_check) {
$basket[0]['link'] = $lnk->href;
$basket[0]['title'] = $lnk->title;
} else {
$can_next = false;
}
}
if ($can_next) {
if ($img) {
$basket[0]['img'] = $img->src;
}
}
}
unset($story_1);
if ($can_next) {
$story_2 = $html->find('li[id="nbm_1"]', 0);
if ($story_2) {
echo "FOUND STORY 2!" . PHP_EOL;
$lnk = $story_2->find('a', 0);
$img = $lnk->find('img', 0);
if ($lnk) {
if ($_DOMAIN . trim($lnk->href) != $uri_to_check) {
$basket[1]['link'] = $lnk->href;
$basket[1]['title'] = $lnk->title;
} else {
$can_next = false;
}
}
if ($can_next) {
if ($img) {
$basket[1]['img'] = $img->src;
}
}
}
unset($story_2);
}
if ($can_next) {
$story_3 = $html->find('li[id="nbm_2"]', 0);
if ($story_3) {
echo "FOUND STORY 3!" . PHP_EOL;
$lnk = $story_3->find('a', 0);
$img = $lnk->find('img', 0);
if ($lnk) {
if ($_DOMAIN . trim($lnk->href) != $uri_to_check) {
$basket[2]['link'] = $lnk->href;
$basket[2]['title'] = $lnk->title;
} else {
$can_next = false;
}
}
if ($can_next) {
if ($img) {
$basket[2]['img'] = $img->src;
}
}
}
unset($story_3);
}
if ($can_next) {
$story_4 = $html->find('li[id="nbm_3"]', 0);
if ($story_4) {
echo "FOUND STORY 4!" . PHP_EOL;
$lnk = $story_4->find('a', 0);
$img = $lnk->find('img', 0);
if ($lnk) {
if ($_DOMAIN . trim($lnk->href) != $uri_to_check) {
$basket[3]['link'] = $lnk->href;
$basket[3]['title'] = $lnk->title;
} else {
$can_next = false;
}
}
if ($can_next) {
if ($img) {
$basket[3]['img'] = $img->src;
}
}
}
unset($story_4);
}
if ($can_next) {
$others = $html->find('div[id="listNews"]', 0);
if ($others) {
echo "BEGIN FIND OTHERS!" . PHP_EOL;
//begin find each story (11 childrens)
//echo count($others->children());
$count = count($others->children());
//$a = 0;
for ($i = 1; $i < $count; $i++) {
if ($can_next) {
//each child contain 2 story;
$st1 = $others->children($i)->children(0);
if ($st1) {
$lnk = $st1->find('a', 1);
$img = $st1->find('img', 0);
if ($lnk) {
if ($_DOMAIN . trim($lnk->href) != $uri_to_check) {
$basket[$i * 3 + 1]['link'] = $lnk->href;
$basket[$i * 3 + 1]['title'] = $lnk->title;
} else {
$can_next = false;
}
}
if ($can_next) {
if ($img) {
$basket[$i * 3 + 1]['img'] = $img->src;
}
}
//echo "FOUND 1: ".$st1->plaintext.PHP_EOL;
//$a++;
}
unset($st1);
$st2 = $others->children($i)->children(1);
if ($st2) {
$lnk = $st2->find('a', 1);
$img = $st2->find('img', 0);
if ($lnk) {
if ($_DOMAIN . trim($lnk->href) != $uri_to_check) {
$basket[$i * 3 + 2]['link'] = $lnk->href;
$basket[$i * 3 + 2]['title'] = $lnk->title;
} else {
$can_next = false;
}
}
if ($can_next) {
if ($img) {
$basket[$i * 3 + 2]['img'] = $img->src;
}
}
//$a++;
//echo "FOUND 2: ".$st2->plaintext.PHP_EOL;
}
unset($st2);
}
//EOF check can go next
}
//EOF for
}
//EOF count others
unset($others);
}
if (isset($lnk)) {
unset($lnk);
}
if (isset($img)) {
unset($img);
}
// begin insert to db
if (count($basket) > 0) {
foreach ($basket as $ball) {
$link = $_DOMAIN . $ball['link'];
if (Model_HoroscopeBLL::CheckStoryByUri($link) == false) {
if (Model_HoroscopeTempBLL::CheckRecordByURI($link) == false) {
$c = new Model_HoroscopeTemp();
$c->title = Vendor_Crawler::unhtmlentities($ball['title']);
$c->uri = $link;
//only get image with width = 140px (max = 360px)
//http://imgthumb.2sao.vietnamnet.vn/ThumbImages/2010/09/13/22/15/A19_360.jpg
$str_arr = explode("/", urldecode($ball['img']));
//pops and returns the last value of the array,
$img_name = array_pop($str_arr);
if ($img_name) {
//A19_360.jpg
list($name, $ext) = explode('.', $img_name);
list($ten_file, $size) = explode('_', $name);
$size = '140';
//set always = 360px
//begin join all part => one
$ss = implode('_', array($ten_file, $size));
//A19_360
$ss = implode('.', array($ss, $ext));
//A19_360.jpg
array_push($str_arr, $ss);
$ball['img'] = implode("/", $str_arr);
}
$c->img = $ball['img'];
$c->save();
unset($c);
$this->_print_to_console(__("INSERT DONE!!!"));
} else {
$this->_print_to_console(__("URL existed, DO NOTHING!!!"));
}
} else {
echo "DO NOTHING!!!" . PHP_EOL;
}
}
} else {
$this->_print_to_console(__("<--- NOTHING TO INSERT --->"));
}
unset($basket);
if ($can_next) {
//find next page link
$pager = $html->find('div[class="pager"]', 0);
if ($pager) {
$links = $pager->find('a');
if (count($links) > 0) {
$cur_page = 0;
for ($i = 0; $i < count($links); $i++) {
if ($links[$i]->class == 'active') {
$cur_page = $i;
}
}
if (isset($links[$cur_page + 1])) {
$next_link = $links[$cur_page + 1]->href;
$next_link = $_DOMAIN . $next_link;
}
}
}
unset($pager);
}
$html->clear();
unset($html);
} else {
//content not found or cant go next
$this->_print_to_console(__("Content not found or cant go next!"));
}
unset($content);
//go to next page
if ($next_link != '') {
$this->_do_craw($_DOMAIN, $next_link, $uri_to_check);
}
}