/** * Craw content from start url to the last url which got * @param string $start_uri: The main start url to check * @param string $md5sum_to_check: The last md5 checksum of url which is have been got * */ public static function _Do_Craw_Content($start_uri, $md5sum_to_check) { self::_print_to_console('Begin fetch URI:<br/>' . $start_uri); $can_next = true; $next_link = ''; $backet = array(); $content = Vendor_Crawler::get_content_from_uri_by_curl($start_uri); if ($content) { self::_print_to_console('Get content done!'); // Create a DOM object require_once Kohana::find_file('classes', 'vendor/simple_html_dom'); $html = new simple_html_dom(); // Load HTML from a string $html->load($content['output']); //begin find content $container = $html->find('div[id="TopicList"]', 0); if ($container) { //begin get story $stories = $container->find('div[class="cont09"]'); if (count($stories) > 0) { //begin parse each story $index = 0; foreach ($stories as $story) { if ($can_next) { # find title,image,summary => send to $backet $ok = true; $str = array(); $img = $story->find('img', 0); if ($img) { $str['image'] = $img->src; } else { $ok = false; } unset($img); $lnk = $story->find('a[class="lnk02"]', 0); if ($lnk) { $str['md5sum'] = md5($lnk->href); if ($str['md5sum'] != $md5sum_to_check) { $str['link'] = $lnk->href; $str['title'] = trim($lnk->plaintext); # $str['cung_1'] = $matches[]; $str['slug'] = Vendor_Crawler::toAscii($str['title']); $pattern = '/([a-z]+[-]+[a-z]+)[-]va[-]([a-z]+[-]+[a-z]+)[-].*/ismUx'; preg_match($pattern, $str['slug'], $matches); if (isset($matches[1]) && isset($matches[2])) { $str['cung_1'] = $matches[1]; $str['cung_2'] = $matches[2]; } else { $pattern = '/hai[-]nguoi[-]cung[-]+([a-z]+[-]+[a-z]+)[-].*/ismUx'; preg_match($pattern, $str['slug'], $matches); if (isset($matches[1])) { $str['cung_1'] = $str['cung_2'] = $matches[1]; } else { $str['cung_1'] = $str['cung_2'] = '-'; } } } else { $can_next = false; } # $str['cung_1'] = $matches; } else { $ok = false; } unset($lnk); if ($can_next) { $summary = $story->find('p[class="cont09txt"]', 0); if ($summary) { $str['summary'] = trim($summary->plaintext); } else { $ok = false; } } if ($ok && $can_next) { $str['date_check'] = date("Y-m-d h:i:s"); $backet[$index] = $str; unset($str); $index++; } } else { return; } } //EOF foreach # get next page link if ($can_next) { $tmp = explode('?p=', $start_uri); if (isset($tmp[1])) { $next_link = $tmp[0] . "?p=" . (intval($tmp[1]) + 1); } else { # next page is the second $next_link = $tmp[0] . "?p=2"; } //echo $next_link; } //var_dump($backet); //insert backet into db if (count($backet > 0)) { $succ = 0; $failure = 0; $dupp = 0; foreach ($backet as $bag) { if (Model_Horoscope_XungHopLinkBLL::CheckRecordByMd5Sum($bag['md5sum'])) { $dupp++; } else { $item = new Model_Horoscope_XungHopLink(); $item->uri = $bag['link']; $item->md5sum = $bag['md5sum']; $item->title = $bag['title']; $item->hinh_anh = $bag['image']; $item->ngay_check = $bag['date_check']; $item->summary = $bag['summary']; $item->cung_1 = $bag['cung_1']; $item->cung_2 = $bag['cung_2']; $item->hoan_thanh = false; $item->save(); if ($item->identifier()) { $succ++; } else { $failure++; } } } } self::_print_to_console('--> INSERT: ' . $succ . ' (ok) , ' . $failure . ' (false) , ' . $dupp . ' (duplicate) <-- <br/>'); flush(); } else { self::_print_to_console('--> No story found!'); $can_next = false; } } else { self::_print_to_console('Cant get main container!'); } $html->clear(); unset($html); unset($content); //sleep a while usleep(2500); //do recursive if ($can_next && $next_link != '') { self::_Do_Craw_Content($next_link, $md5sum_to_check); } } else { self::_print_to_console('Cant get content for:<br/>' . $start_uri); } }