/**
  * 抓取页面数据
  */
 private function doSpiderContent()
 {
     $s_sql_link = 'SELECT id,catid,cattitle,subcatid,subcattitle,title,area,summary,url ' . 'FROM pre_sale_spider_links WHERE state=0 ORDER BY id asc';
     $r_result = DB::query($s_sql_link);
     $a_link_list = array();
     while ($tem = DB::fetch($r_result)) {
         $a_link_list[] = $tem;
     }
     $a_config_temp = array();
     foreach ($a_link_list as $a_link) {
         //日志
         $this->addLog($a_link['url'], true);
         $a_config_temp['catId'] = $a_link['catid'];
         $a_config_temp['catTitle'] = $a_link['cattitle'];
         $a_config_temp['subCatId'] = $a_link['subcatid'];
         $a_config_temp['subCatTitle'] = $a_link['subcattitle'];
         $a_config_temp['area'] = $a_link['area'];
         $a_config_temp['desc'] = $a_link['summary'];
         $o_operation = new SalePageData();
         $b_flag = $o_operation->doParse($a_link['url'], $a_config_temp);
         //解析失败
         if ($b_flag == false) {
             DB::update('sale_spider_links', array('state' => 2), "id={$a_link['id']}");
             //日志
             $this->addLog('doParse Error');
             continue;
         }
         $a_goods_values = $o_operation->getArrayData();
         $s_goods_id = DB::insert('sale_goods', $a_goods_values, $s_goods_id = true);
         $a_member_values = $o_operation->getUserDataArr($s_goods_id);
         DB::insert('sale_member', $a_member_values);
         DB::update('sale_spider_links', array('state' => 1), "id={$a_link['id']}");
         //日志
         $this->addLog("id:{$s_goods_id},catid:{$a_link['subcatid']},title:{$a_link['title']}");
         //暂停N秒
         //            sleep($this->_i_wait_second);
     }
 }
Beispiel #2
0
     $a_config_temp = array();
     $ii = 0;
     foreach ($a_link_list as $a_link) {
         if (++$i_spider_count > $_SPIDER_CONTENT_ONCE) {
             break;
         }
         sale_spider_content_debug($a_link, $i_spider_count);
         ob_flush();
         flush();
         $a_config_temp['catId'] = $a_link['catid'];
         $a_config_temp['catTitle'] = $a_link['cattitle'];
         $a_config_temp['subCatId'] = $a_link['subcatid'];
         $a_config_temp['subCatTitle'] = $a_link['subcattitle'];
         $a_config_temp['area'] = $a_link['area'];
         $a_config_temp['desc'] = $a_link['summary'];
         $o_operation = new SalePageData();
         $b_flag = $o_operation->doParse($a_link['url'], $a_config_temp);
         //解析失败
         if ($b_flag == false) {
             DB::update('sale_spider_links', array('state' => 2), "id={$a_link['id']}");
             continue;
         }
         $a_goods_values = $o_operation->getArrayData();
         $s_goods_id = DB::insert('sale_goods', $a_goods_values, $s_goods_id = true);
         $a_member_values = $o_operation->getUserDataArr($s_goods_id);
         DB::insert('sale_member', $a_member_values);
         DB::update('sale_spider_links', array('state' => 1), "id={$a_link['id']}");
         echo '===================================================<br/><br/>';
     }
     include template("sale:admin/admin_spider_content");
 } else {