public function all($domain_id = null, $limit = null, $offset = null, $new = true)
{
log_message('info', 'Fetch : ALL');
$option = array('outdate' => 0, 'active_score >' => -10);
if ($new) {
$option['parse_post'] = 0;
}
if ($domain_id != null) {
$option['domain_id'] = $domain_id;
}
if ($offset != null && $limit != null) {
echo "LIMITE = {$limit}, OFFSET = {$offset}\n";
$this->db->limit($limit, $offset);
}
$this->db->order_by('active_score', 'desc');
$query = $this->db->get_where('page', $option);
echo "FETCH ({$query->num_rows}) : Page ";
$write_file = false;
if ($query->num_rows() > 0) {
log_message('info', 'Fetch : found : ' . $query->num_rows() . " rows.");
foreach ($query->result() as $row) {
// Reset PHP Timeout to 1min
set_time_limit(600);
$page = new Page_model();
$page->init($row->id);
echo ',' . $page->id;
$fetch = $page->fetch();
if ($fetch != null) {
if ($page->size == null) {
echo "(same)";
$res = $page->update_same_page($fetch);
if (!$res) {
echo PHP_EOL . "write file error";
exit;
}
$res = $page->parse($fetch['content']);
if ($res) {
// parse success -> collect data from mememcache and inset to db
}
} else {
if ($page->parent_page_id == 0) {
// has update
echo "(root)";
$page->update_new_page($fetch, true);
$page->more_active();
$res = $page->parse($fetch['content']);
if ($res) {
// parse success -> collect data from mememcache and inset to db
}
} else {
if ($page->parent_page_id != 0 && $this->compare_size($page, $fetch) > 500) {
// has update
echo "(+new)";
$page->update_new_page($fetch, true);
$page->more_active();
$res = $page->parse($fetch['content']);
if ($res) {
// parse success -> collect data from mememcache and inset to db
}
} else {
echo "(noch)";
// $page->parse($fetch['content']);
$page->less_active();
}
}
}
} else {
echo "(-err)";
$page->less_active();
}
$page->update();
unset($page);
}
}
}
public function all($domain_id = null, $limit = null, $offset = null, $new = true)
{
log_message('info', 'Fetch : ALL');
$option = array('outdate' => 0, 'active_score >' => -10);
if ($new) {
$option['parse_post'] = 0;
}
if ($domain_id != null) {
$option['domain_id'] = $domain_id;
}
if ($offset != null && $limit != null) {
echo "LIMITE = {$limit}, OFFSET = {$offset}\n";
$this->db->limit($limit, $offset);
}
$this->db->order_by('active_score', 'desc');
$query = $this->db->get_where('page', $option);
echo "FETCH ({$query->num_rows}) : Page ";
$write_file = false;
if ($query->num_rows() > 0) {
log_message('info', 'Fetch : found : ' . $query->num_rows() . " rows.");
foreach ($query->result() as $row) {
// Reset PHP Timeout to 1min
set_time_limit(600);
$page = new Page_model();
$page->init($row->id);
echo ',' . $page->id;
$fetch = $page->fetch();
$res = null;
if ($fetch != null) {
if ($page->size == null) {
echo "(same)";
$res = $page->parse($fetch['content']);
} else {
if ($page->parent_page_id == 0) {
// has update
echo "(root)";
} else {
if ($page->parent_page_id != 0 && $this->compare_size($page, $fetch) > 500) {
// has update
echo "(+new)";
$res = $page->parse($fetch['content']);
} else {
echo "(noch)";
$page->less_active();
}
}
}
} else {
echo "(-err)";
$page->less_active();
}
if ($res != null) {
if (!$res['parse_ok']) {
$page->parse_post = -1;
} else {
// batch insert posts
$list = array();
foreach ($res['posts'] as $post) {
$list[] = (array) $post;
}
//var_dump($list);
$insert_res = $this->db->insert_batch('post', $list);
// if batch insert success
// $page->parse_post = 1
// update page latest_fetch
// or failed
// $page->parse_post = -2
if ($insert_res == true) {
$page->parse_post = 1;
$this->lastest_fetch = mdate('%Y-%m-%d %H:%i', time());
$page->size = $fetch['size'];
// $page->latest_fetch = ??
// $page->size = ??
} else {
$page->parse_post = -2;
}
//unset($page);
}
}
if ($page->parse_post < 0) {
// write file
// ถ้า page มีไฟล์อยู่แล้ว
// ลบไฟล์เก่า ก่อน
// เขียนไฟล์ใหม่
//filename = latest_fetch + id
$filename = mdate('%Y%m%d%H%i', time()) . "_" . $page->id;
//file_folder = 8 digits Year+Month+Day
$folder = mdate('%Y%m%d', time()) . '/';
//$path = $this->config->item('fetch_file_path');
$path = $this->config->item('fetch_file_path');
if (file_exists($path . $folder . $filename)) {
unlink($path . $folder . $filename);
}
write_file($path . $folder . $filename, $fetch['content']);
}
$page->update();
unset($page);
}
}
}
