public function all($domain_id = null, $limit = null, $offset = null, $new = true) { log_message('info', 'Fetch : ALL'); $option = array('outdate' => 0, 'active_score >' => -10); if ($new) { $option['parse_post'] = 0; } if ($domain_id != null) { $option['domain_id'] = $domain_id; } if ($offset != null && $limit != null) { echo "LIMITE = {$limit}, OFFSET = {$offset}\n"; $this->db->limit($limit, $offset); } $this->db->order_by('active_score', 'desc'); $query = $this->db->get_where('page', $option); echo "FETCH ({$query->num_rows}) : Page "; $write_file = false; if ($query->num_rows() > 0) { log_message('info', 'Fetch : found : ' . $query->num_rows() . " rows."); foreach ($query->result() as $row) { // Reset PHP Timeout to 1min set_time_limit(600); $page = new Page_model(); $page->init($row->id); echo ',' . $page->id; $fetch = $page->fetch(); if ($fetch != null) { if ($page->size == null) { echo "(same)"; $res = $page->update_same_page($fetch); if (!$res) { echo PHP_EOL . "write file error"; exit; } $res = $page->parse($fetch['content']); if ($res) { // parse success -> collect data from mememcache and inset to db } } else { if ($page->parent_page_id == 0) { // has update echo "(root)"; $page->update_new_page($fetch, true); $page->more_active(); $res = $page->parse($fetch['content']); if ($res) { // parse success -> collect data from mememcache and inset to db } } else { if ($page->parent_page_id != 0 && $this->compare_size($page, $fetch) > 500) { // has update echo "(+new)"; $page->update_new_page($fetch, true); $page->more_active(); $res = $page->parse($fetch['content']); if ($res) { // parse success -> collect data from mememcache and inset to db } } else { echo "(noch)"; // $page->parse($fetch['content']); $page->less_active(); } } } } else { echo "(-err)"; $page->less_active(); } $page->update(); unset($page); } } }
public function all($domain_id = null, $limit = null, $offset = null, $new = true) { log_message('info', 'Fetch : ALL'); $option = array('outdate' => 0, 'active_score >' => -10); if ($new) { $option['parse_post'] = 0; } if ($domain_id != null) { $option['domain_id'] = $domain_id; } if ($offset != null && $limit != null) { echo "LIMITE = {$limit}, OFFSET = {$offset}\n"; $this->db->limit($limit, $offset); } $this->db->order_by('active_score', 'desc'); $query = $this->db->get_where('page', $option); echo "FETCH ({$query->num_rows}) : Page "; $write_file = false; if ($query->num_rows() > 0) { log_message('info', 'Fetch : found : ' . $query->num_rows() . " rows."); foreach ($query->result() as $row) { // Reset PHP Timeout to 1min set_time_limit(600); $page = new Page_model(); $page->init($row->id); echo ',' . $page->id; $fetch = $page->fetch(); $res = null; if ($fetch != null) { if ($page->size == null) { echo "(same)"; $res = $page->parse($fetch['content']); } else { if ($page->parent_page_id == 0) { // has update echo "(root)"; } else { if ($page->parent_page_id != 0 && $this->compare_size($page, $fetch) > 500) { // has update echo "(+new)"; $res = $page->parse($fetch['content']); } else { echo "(noch)"; $page->less_active(); } } } } else { echo "(-err)"; $page->less_active(); } if ($res != null) { if (!$res['parse_ok']) { $page->parse_post = -1; } else { // batch insert posts $list = array(); foreach ($res['posts'] as $post) { $list[] = (array) $post; } //var_dump($list); $insert_res = $this->db->insert_batch('post', $list); // if batch insert success // $page->parse_post = 1 // update page latest_fetch // or failed // $page->parse_post = -2 if ($insert_res == true) { $page->parse_post = 1; $this->lastest_fetch = mdate('%Y-%m-%d %H:%i', time()); $page->size = $fetch['size']; // $page->latest_fetch = ?? // $page->size = ?? } else { $page->parse_post = -2; } //unset($page); } } if ($page->parse_post < 0) { // write file // ถ้า page มีไฟล์อยู่แล้ว // ลบไฟล์เก่า ก่อน // เขียนไฟล์ใหม่ //filename = latest_fetch + id $filename = mdate('%Y%m%d%H%i', time()) . "_" . $page->id; //file_folder = 8 digits Year+Month+Day $folder = mdate('%Y%m%d', time()) . '/'; //$path = $this->config->item('fetch_file_path'); $path = $this->config->item('fetch_file_path'); if (file_exists($path . $folder . $filename)) { unlink($path . $folder . $filename); } write_file($path . $folder . $filename, $fetch['content']); } $page->update(); unset($page); } } }