/** * 抓取页面数据 */ private function doSpiderContent() { $s_sql_link = 'SELECT id,catid,cattitle,subcatid,subcattitle,title,area,summary,url ' . 'FROM pre_sale_spider_links WHERE state=0 ORDER BY id asc'; $r_result = DB::query($s_sql_link); $a_link_list = array(); while ($tem = DB::fetch($r_result)) { $a_link_list[] = $tem; } $a_config_temp = array(); foreach ($a_link_list as $a_link) { //日志 $this->addLog($a_link['url'], true); $a_config_temp['catId'] = $a_link['catid']; $a_config_temp['catTitle'] = $a_link['cattitle']; $a_config_temp['subCatId'] = $a_link['subcatid']; $a_config_temp['subCatTitle'] = $a_link['subcattitle']; $a_config_temp['area'] = $a_link['area']; $a_config_temp['desc'] = $a_link['summary']; $o_operation = new SalePageData(); $b_flag = $o_operation->doParse($a_link['url'], $a_config_temp); //解析失败 if ($b_flag == false) { DB::update('sale_spider_links', array('state' => 2), "id={$a_link['id']}"); //日志 $this->addLog('doParse Error'); continue; } $a_goods_values = $o_operation->getArrayData(); $s_goods_id = DB::insert('sale_goods', $a_goods_values, $s_goods_id = true); $a_member_values = $o_operation->getUserDataArr($s_goods_id); DB::insert('sale_member', $a_member_values); DB::update('sale_spider_links', array('state' => 1), "id={$a_link['id']}"); //日志 $this->addLog("id:{$s_goods_id},catid:{$a_link['subcatid']},title:{$a_link['title']}"); //暂停N秒 // sleep($this->_i_wait_second); } }
$a_config_temp = array(); $ii = 0; foreach ($a_link_list as $a_link) { if (++$i_spider_count > $_SPIDER_CONTENT_ONCE) { break; } sale_spider_content_debug($a_link, $i_spider_count); ob_flush(); flush(); $a_config_temp['catId'] = $a_link['catid']; $a_config_temp['catTitle'] = $a_link['cattitle']; $a_config_temp['subCatId'] = $a_link['subcatid']; $a_config_temp['subCatTitle'] = $a_link['subcattitle']; $a_config_temp['area'] = $a_link['area']; $a_config_temp['desc'] = $a_link['summary']; $o_operation = new SalePageData(); $b_flag = $o_operation->doParse($a_link['url'], $a_config_temp); //解析失败 if ($b_flag == false) { DB::update('sale_spider_links', array('state' => 2), "id={$a_link['id']}"); continue; } $a_goods_values = $o_operation->getArrayData(); $s_goods_id = DB::insert('sale_goods', $a_goods_values, $s_goods_id = true); $a_member_values = $o_operation->getUserDataArr($s_goods_id); DB::insert('sale_member', $a_member_values); DB::update('sale_spider_links', array('state' => 1), "id={$a_link['id']}"); echo '===================================================<br/><br/>'; } include template("sale:admin/admin_spider_content"); } else {