/** * 抓取资源 * @param [string] $html [抓取结果] * @param [array] $data [数据项] * @param [array] $rule [规则] * @param [array] $responses [已经抓取资源] * @return [array] [返回处理结果] */ public static function crawl($html, $data, $rule, $responses) { if (trim($data['rule']) === '') { return; } $name = $data['name']; if (spider::$dataTest) { print_r('<b>[' . $name . ']规则:</b>' . iS::escapeStr($data['rule'])); echo "<hr />"; } if (strpos($data['rule'], 'RULE@') !== false) { spider::$rid = str_replace('RULE@', '', $data['rule']); $_urls = trim($html); if (spider::$dataTest) { print_r('<b>使用[rid:' . spider::$rid . ']规则抓取</b>:' . $_urls); echo "<hr />"; } return spiderUrls::crawl('DATA@RULE', false, spider::$rid, $_urls); } /** * RAND@10,0 * 返回随机数 */ if (strpos($data['rule'], 'RAND@') !== false) { $random = str_replace('RAND@', '', $data['rule']); list($length, $numeric) = explode(',', $random); return random($length, empty($numeric) ? 0 : 1); } $contentArray = array(); $contentHash = array(); $_content = null; $_content = spiderContent::match($html, $data, $rule); $cmd5 = md5($_content); $contentArray[] = $_content; $contentHash[$cmd5] = true; if ($data['page']) { if (empty($rule['page_url'])) { $rule['page_url'] = $rule['list_url']; } if (empty(spider::$allHtml)) { $page_url_array = array(); $page_area_rule = trim($rule['page_area_rule']); if ($page_area_rule) { if (strpos($page_area_rule, 'DOM::') !== false) { iPHP::import(iPHP_LIB . '/phpQuery.php'); $doc = phpQuery::newDocumentHTML($html, 'UTF-8'); $pq_dom = str_replace('DOM::', '', $page_area_rule); $pq_array = phpQuery::pq($pq_dom); foreach ($pq_array as $pn => $pq_val) { $href = phpQuery::pq($pq_val)->attr('href'); if ($href) { if ($rule['page_url_rule']) { if (strpos($rule['page_url_rule'], '<%') !== false) { $page_url_rule = spiderTools::pregTag($rule['page_url_rule']); if (!preg_match('|' . $page_url_rule . '|is', $href)) { continue; } } else { $cleanhref = spiderTools::dataClean($rule['page_url_rule'], $href); if ($cleanhref) { $href = $cleanhref; unset($cleanhref); } else { continue; } } } $href = str_replace('<%url%>', $href, $rule['page_url']); $page_url_array[$pn] = spiderTools::url_complement($rule['__url__'], $href); } } phpQuery::unloadDocuments($doc->getDocumentID()); } else { $page_area_rule = spiderTools::pregTag($page_area_rule); if ($page_area_rule) { preg_match('|' . $page_area_rule . '|is', $html, $matches, $PREG_SET_ORDER); $page_area = $matches['content']; } else { $page_area = $html; } if ($rule['page_url_rule']) { $page_url_rule = spiderTools::pregTag($rule['page_url_rule']); preg_match_all('|' . $page_url_rule . '|is', $page_area, $page_url_matches, PREG_SET_ORDER); foreach ($page_url_matches as $pn => $row) { $href = str_replace('<%url%>', $row['url'], $rule['page_url']); $page_url_array[$pn] = spiderTools::url_complement($rule['__url__'], $href); gc_collect_cycles(); } } unset($page_area); } } else { // 逻辑方式 if ($rule['page_url_parse'] == '<%url%>') { $page_url = str_replace('<%url%>', $rule['__url__'], $rule['page_url']); } else { $page_url_rule = spiderTools::pregTag($rule['page_url_parse']); preg_match('|' . $page_url_rule . '|is', $rule['__url__'], $matches, $PREG_SET_ORDER); $page_url = str_replace('<%url%>', $matches['url'], $rule['page_url']); } if (stripos($page_url, '<%step%>') !== false) { for ($pn = $rule['page_no_start']; $pn <= $rule['page_no_end']; $pn = $pn + $rule['page_no_step']) { $page_url_array[$pn] = str_replace('<%step%>', $pn, $page_url); gc_collect_cycles(); } } } //URL去重清理 if ($page_url_array) { $page_url_array = array_filter($page_url_array); $page_url_array = array_unique($page_url_array); $puk = array_search($rule['__url__'], $page_url_array); if ($puk !== false) { unset($page_url_array[$puk]); } } if (spider::$dataTest) { echo "<b>内容页网址:</b>" . $rule['__url__'] . "<br />"; echo "<b>分页:</b>" . $rule['page_url'] . "<br />"; echo iS::escapeStr($page_url_rule); echo "<hr />"; } if (spider::$dataTest) { echo "<b>分页列表:</b><pre>"; print_r($page_url_array); echo "</pre><hr />"; } spider::$content_right_code = trim($rule['page_url_right']); spider::$content_error_code = trim($rule['page_url_error']); spider::$curl_proxy = $rule['proxy']; $pageurl = array(); foreach ($page_url_array as $pukey => $purl) { //usleep(100); $phtml = spiderTools::remote($purl); if (empty($phtml)) { break; } $md5 = md5($phtml); if ($pageurl[$md5]) { break; } $check_content = spiderTools::check_content_code($phtml); if ($check_content === false) { unset($check_content, $phtml); break; } $_content = spiderContent::match($phtml, $data, $rule); $cmd5 = md5($_content); if ($contentHash[$cmd5]) { break; } $contentArray[] = $_content; $contentHash[$cmd5] = true; $pageurl[$md5] = $purl; spider::$allHtml[$md5] = $phtml; } gc_collect_cycles(); unset($check_content, $phtml); if (spider::$dataTest) { echo "<b>最终分页列表:</b><pre>"; print_r($pageurl); echo "</pre><hr />"; } } else { foreach ((array) spider::$allHtml as $ahkey => $phtml) { $contentArray[] = spiderContent::match($phtml, $data, $rule); } } } $content = implode('#--iCMS.PageBreak--#', $contentArray); $html = null; unset($html, $contentArray, $contentHash, $_content); $content = stripslashes($content); if (spider::$dataTest) { print_r('<b>[' . $name . ']匹配结果:</b>' . htmlspecialchars($content)); echo "<hr />"; } if ($data['cleanbefor']) { $content = spiderTools::dataClean($data['cleanbefor'], $content); } /** * 在数据项里调用之前采集的数据[DATA@name][DATA@name.key] */ if (strpos($content, '[DATA@') !== false) { $content = spiderTools::getDATA($responses, $content); } if ($data['cleanhtml']) { $content = stripslashes($content); $content = preg_replace('/<[\\/\\!]*?[^<>]*?>/is', '', $content); } if ($data['format'] && $content) { $content = autoformat($content); } if ($data['img_absolute'] && $content) { // $content = stripslashes($content); preg_match_all("/<img.*?src\\s*=[\"|'](.*?)[\"|']/is", $content, $img_match); if ($img_match[1]) { $_img_array = array_unique($img_match[1]); $_img_urls = array(); foreach ((array) $_img_array as $_img_key => $_img_src) { $_img_urls[$_img_key] = spiderTools::url_complement($rule['__url__'], $_img_src); } $content = str_replace($_img_array, $_img_urls, $content); } unset($img_match, $_img_array, $_img_urls, $_img_src); } if ($data['trim']) { $content = trim($content); } if ($data['capture']) { // $content = stripslashes($content); $content = spiderTools::remote($content); } if ($data['download']) { // $content = stripslashes($content); $content = iFS::http($content); } if ($data['cleanafter']) { $content = spiderTools::dataClean($data['cleanafter'], $content); // $content = stripslashes($content); } if ($data['autobreakpage']) { $content = spiderTools::autoBreakPage($content); } if ($data['mergepage']) { $content = spiderTools::mergePage($content); } if ($data['empty'] && empty($content)) { $emptyMsg = '[' . $name . ']规则设置了不允许为空.当前抓取结果为空!请检查,规则是否正确!'; if (spider::$dataTest) { exit('<h1>' . $emptyMsg . '</h1>'); } if (spider::$work) { echo "\n{$emptyMsg}\n"; return false; } else { iPHP::alert($emptyMsg); } } if ($data['json_decode']) { $content = json_decode($content, true); } if ($data['array']) { return (array) $content; } if (spider::$callback['content'] && is_callable(spider::$callback['content'])) { $content = call_user_func_array(spider::$callback['content'], array($content)); } return $content; }
public static function crawl($work = NULL, $pid = NULL, $_rid = NULL, $_urls = null, $callback = null) { $pid === NULL && ($pid = spider::$pid); if ($pid) { $project = spider::project($pid); $cid = $project['cid']; $rid = $project['rid']; $prule_list_url = $project['list_url']; $lastupdate = $project['lastupdate']; } else { $cid = spider::$cid; $rid = spider::$rid; } if (empty($rid) && $_rid !== NULL) { $rid = $_rid; } if ($work == 'shell') { $lastupdate = $project['lastupdate']; if ($project['psleep']) { if (time() - $lastupdate < $project['psleep']) { echo '采集方案[' . $pid . "]:" . format_date($lastupdate) . "刚采集过了,请" . $project['psleep'] / 3600 . "小时后在继续采集\n"; return; } } echo "[32m开始采集方案[" . $pid . "] 采集规则[" . $rid . "][0m\n"; } $ruleA = spider::rule($rid); $rule = $ruleA['rule']; $urls = $rule['list_urls']; $project['urls'] && ($urls = $project['urls']); spiderUrls::$urls && ($urls = spiderUrls::$urls); $_urls && ($urls = $_urls); $urlsArray = explode("\n", $urls); $urlsArray = array_filter($urlsArray); $_urlsArray = $urlsArray; $urlsList = array(); if ($work == 'shell') { // echo "$urls\n"; print_r($urlsArray); } foreach ($_urlsArray as $_key => $_url) { $_url = htmlspecialchars_decode($_url); $_urlsList = array(); /** * RULE@rid@url * url使用[rid]规则采集并返回列表结果 */ if (strpos($_url, 'RULE@') !== false) { list($___s, $_rid, $_urls) = explode('@', $_url); if (spider::$ruleTest) { print_r('<b>使用[rid:' . $_rid . ']规则抓取列表</b>:' . $_urls); echo "<hr />"; } $_urlsList = spiderUrls::crawl($work, false, $_rid, $_urls, 'CALLBACK@URL'); $urlsList = array_merge($urlsList, $_urlsList); unset($urlsArray[$_key]); } else { preg_match('|.*<(.*)>.*|is', $_url, $_matches); if ($_matches) { list($format, $begin, $num, $step, $zeroize, $reverse) = explode(',', $_matches[1]); $url = str_replace($_matches[1], '*', trim($_matches[0])); $_urlsList = spiderTools::mkurls($url, $format, $begin, $num, $step, $zeroize, $reverse); unset($urlsArray[$_key]); $urlsList = array_merge($urlsList, $_urlsList); } } } $urlsList && ($urlsArray = array_merge($urlsArray, $urlsList)); unset($_urlsArray, $_key, $_url, $_matches, $_urlsList, $urlsList); $urlsArray = array_unique($urlsArray); // spider::$useragent = $rule['user_agent']; // spider::$encoding = $rule['curl']['encoding']; // spider::$referer = $rule['curl']['referer']; // spider::$charset = $rule['charset']; if (empty($urlsArray)) { if ($work == 'shell') { echo "采集列表为空!请填写!\n"; return false; } iPHP::alert('采集列表为空!请填写!', 'js:parent.window.iCMS_MODAL.destroy();'); } // if(spider::$ruleTest){ // echo "<pre>"; // print_r(iS::escapeStr($project)); // print_r(iS::escapeStr($rule)); // echo "</pre>"; // echo "<hr />"; // } if ($rule['mode'] == "2") { iPHP::import(iPHP_LIB . '/phpQuery.php'); spider::$ruleTest && $_GET['pq_debug'] && (phpQuery::$debug = 1); } $pubArray = array(); $pubCount = array(); $pubAllCount = array(); spider::$curl_proxy = $rule['proxy']; spider::$urlslast = null; foreach ($urlsArray as $key => $url) { $url = trim($url); spider::$urlslast = $url; if ($work == 'shell') { echo '开始采集列表:' . $url . "\n"; } if (spider::$ruleTest) { echo '<b>抓取列表:</b>' . $url . "<br />"; } $html = spiderTools::remote($url); if (empty($html)) { continue; } if ($rule['mode'] == "2") { $doc = phpQuery::newDocumentHTML($html, 'UTF-8'); $list_area = $doc[trim($rule['list_area_rule'])]; // if(strpos($rule['list_area_format'], 'DOM::')!==false){ // $list_area = spiderTools::dataClean($rule['list_area_format'], $list_area); // } if ($rule['list_area_format']) { $list_area_format = trim($rule['list_area_format']); if (strpos($list_area_format, 'ARRAY::') !== false) { $list_area_format = str_replace('ARRAY::', '', $list_area_format); $lists = array(); foreach ($list_area as $la_key => $la) { $lists[] = phpQuery::pq($list_area_format, $la); } } else { $lists = phpQuery::pq($list_area_format, $list_area); } } else { $lists = $list_area; } // $lists = $list_area; //echo 'list:getDocumentID:'.$lists->getDocumentID()."\n"; } else { $list_area_rule = spiderTools::pregTag($rule['list_area_rule']); if ($list_area_rule) { preg_match('|' . $list_area_rule . '|is', $html, $matches, $PREG_SET_ORDER); $list_area = $matches['content']; } else { $list_area = $html; } $html = null; unset($html); if (spider::$ruleTest) { echo iS::escapeStr($rule['list_area_rule']); // echo iS::escapeStr($list_area); echo "<hr />"; } if ($rule['list_area_format']) { $list_area = spiderTools::dataClean($rule['list_area_format'], $list_area); } preg_match_all('|' . spiderTools::pregTag($rule['list_url_rule']) . '|is', $list_area, $lists, PREG_SET_ORDER); $list_area = null; unset($list_area); if ($rule['sort'] == "1") { //arsort($lists); } elseif ($rule['sort'] == "2") { asort($lists); } elseif ($rule['sort'] == "3") { shuffle($lists); } } if (spider::$ruleTest) { echo '<b>列表区域规则:</b>' . iS::escapeStr($rule['list_area_rule']); echo "<hr />"; echo '<b>列表区域抓取结果:</b>' . iS::escapeStr($list_area); echo "<hr />"; echo '<b>列表链接规则:</b>' . iS::escapeStr($rule['list_url_rule']); echo "<hr />"; echo '<b>网址合成规则:</b>' . iS::escapeStr($rule['list_url']); echo "<hr />"; } if ($prule_list_url) { $rule['list_url'] = $prule_list_url; } //PID@xx 返回URL列表 if ($callback == 'CALLBACK@URL') { $cbListUrl = array(); foreach ($lists as $lkey => $row) { list(spider::$title, spider::$url) = spiderTools::title_url($row, $rule, $url); if (spider::$url === false) { continue; } // if(spider::checker($work)===true){ $cbListUrl[] = spider::$url; // } } return $cbListUrl; } if ($work == "shell") { $pubCount[$url]['count'] = count($lists); $pubAllCount['count'] += $pubCount[$url]['count']; echo "开始采集:" . $url . " 列表 " . $pubCount[$url]['count'] . "条记录\n"; foreach ($lists as $lkey => $row) { list(spider::$title, spider::$url) = spiderTools::title_url($row, $rule, $url); if (spider::$url === false) { continue; } $hash = md5(spider::$url); echo "title:" . spider::$title . "\n"; echo "url:" . spider::$url . "\n"; spider::$rid = $rid; $checker = spider::checker($work); if ($checker === true) { echo "开始采集...."; $callback = spider::publish("shell"); if ($callback['code'] == "1001") { $pubCount[$url]['success']++; $pubAllCount['success']++; echo "....√\n"; if ($project['sleep']) { echo "sleep:" . $project['sleep'] . "s\n"; if ($rule['mode'] != "2") { unset($lists[$lkey]); } gc_collect_cycles(); sleep($project['sleep']); } else { //sleep(1); } } else { $pubCount[$url]['error']++; $pubAllCount['error']++; echo "error\n\n"; continue; } } $pubCount[$url]['published']++; $pubAllCount['published']++; } if ($rule['mode'] == "2") { phpQuery::unloadDocuments($doc->getDocumentID()); } else { unset($lists); } } if ($work == "WEB@MANUAL") { $listsArray[$url] = $lists; } if ($work == "WEB@AUTO" || $work == 'DATA@RULE') { foreach ($lists as $lkey => $row) { list(spider::$title, spider::$url) = spiderTools::title_url($row, $rule, $url); if (spider::$url === false) { continue; } $hash = md5(spider::$url); if (spider::$ruleTest) { echo '<b>列表抓取结果:</b>' . $lkey . '<br />'; echo spider::$title . ' (<a href="' . APP_URI . '&do=testdata' . '&url=' . urlencode(spider::$url) . '&rid=' . $rid . '&pid=' . $pid . '&title=' . urlencode(spider::$title) . '" target="_blank">测试内容规则</a>) <br />'; echo spider::$url . "<br />"; echo $hash . "<br /><hr />"; } else { if (spider::checker($work) === true || spider::$dataTest) { $suData = array('sid' => 0, 'url' => spider::$url, 'title' => spider::$title, 'cid' => $cid, 'rid' => $rid, 'pid' => $pid, 'hash' => $hash); switch ($work) { case 'DATA@RULE': $contentArray[$lkey] = spiderData::crawl(); // $contentArray[$lkey] = spiderUrls::crawl($work,$_pid); unset($suData['sid']); $suData['title'] = addslashes($suData['title']); $suData += array('addtime' => time(), 'status' => '2', 'publish' => '2', 'indexid' => '0', 'pubdate' => '0'); spider::$dataTest or $suid = iDB::insert('spider_url', $suData); $contentArray[$lkey]['spider_url'] = $suid; break; case 'WEB@AUTO': $pubArray[] = $suData; break; } } } } } } $lists = null; unset($lists); gc_collect_cycles(); switch ($work) { case 'WEB@AUTO': return $pubArray; break; case 'DATA@RULE': return $contentArray; break; case 'WEB@MANUAL': return array('cid' => $cid, 'rid' => $rid, 'pid' => $pid, 'sid' => $sid, 'work' => $work, 'rule' => $rule, 'listsArray' => $listsArray); break; case "shell": echo "采集数据统结果:\n"; print_r($pubCount); print_r($pubAllCount); echo "全部采集完成....\n"; iDB::update('spider_project', array('lastupdate' => time()), array('id' => $pid)); break; } }
public static function title_url($row, $rule, $baseUrl = null) { spiderTools::$listArray = array(); $responses = array(); if (strpos($rule['list_url_rule'], '<%url%>') !== false) { $responses = $row; } else { if (is_object($row)) { $list_url_rule = explode("\n", $rule['list_url_rule']); $DOM = phpQuery::pq($row); $keyMap = array('title', 'url'); foreach ($list_url_rule as $key => $value) { $dom_rule = trim($value); if (strpos($dom_rule, '@@') !== false) { list($dom_key, $dom_rule) = explode("@@", $dom_rule); } else { $dom_key = $keyMap[$key]; } $content = ''; if (strpos($dom_rule, 'DOM::') !== false) { $content = spiderTools::domAttr($DOM, $dom_rule); } else { if ($dom_key == 'url') { $dom_rule or $dom_rule = 'href'; } if ($dom_key == 'title') { $dom_rule or $dom_rule = 'text'; } if ($dom_rule == 'text') { $content = $DOM->text(); } else { $content = $DOM->attr($dom_rule); } } $responses[$dom_key] = str_replace(' ', '', trim($content)); } unset($DOM); } } $title = $responses['title']; $url = $responses['url']; $title = trim($title); $url = trim($url); $url = str_replace('<%url%>', $url, $rule['list_url']); if (strpos($url, 'AUTO::') !== false && $baseUrl) { $url = str_replace('AUTO::', '', $url); $url = spiderTools::url_complement($baseUrl, $url); } $rule['list_url_clean'] && ($url = spiderTools::dataClean($rule['list_url_clean'], $url)); $title = preg_replace('/<[\\/\\!]*?[^<>]*?>/is', '', $title); unset($responses['title'], $responses['url']); if ($responses) { foreach ($responses as $key => $value) { if (!is_numeric($key) && strpos($key, 'var_') === false) { spiderTools::$listArray[$key] = $value; } } unset($responses); } return array($title, $url); }
public static function title_url($row, $rule, $baseUrl = null) { if (strpos($rule['list_url_rule'], '<%url%>') !== false) { $title = $row['title']; $url = $row['url']; } else { if (is_object($row)) { list($title_dom, $url_dom) = explode("\n", $rule['list_url_rule']); $DOM = phpQuery::pq($row); $title_dom = trim($title_dom); $url_dom = trim($url_dom); if (strpos($title_dom, 'DOM::') !== false) { $title = spiderTools::domAttr($DOM, $title_dom); } else { $title_dom or $title_dom = 'text'; if ($title_dom == 'text') { $title = $DOM->text(); } else { $title = $DOM->attr($title_dom); } } if (strpos($url_dom, 'DOM::') !== false) { $url = spiderTools::domAttr($DOM, $url_dom); } else { $url_dom or $url_dom = 'href'; $url = $DOM->attr($url_dom); } unset($DOM, $title_dom, $url_dom); } } $title = trim($title); $url = trim($url); $url = str_replace('<%url%>', $url, $rule['list_url']); if (strpos($url, 'AUTO::') !== false && $baseUrl) { $url = str_replace('AUTO::', '', $url); $url = spiderTools::url_complement($baseUrl, $url); } $rule['list_url_clean'] && ($url = spiderTools::dataClean($rule['list_url_clean'], $url)); $title = preg_replace('/<[\\/\\!]*?[^<>]*?>/is', '', $title); // $this->title = $title; return array($title, $url); }