/**
* 强制更新缓存
*/
public function updateCache()
{
//
self::$__tasks = null;
//
self::$__tasks = $this->readTaskArray();
if (isset($this->cache)) {
$this->cache->set('collect.task', $__tasks);
}
return true;
}
public function loadCharsetAction($id)
{
$template = CollectTemplateModel::getTemplateById($id, false);
$_charset = CollectTaskModel::getCharset();
echo $_charset[$template['collect_template_charset']];
}
public function collectAction($id)
{
$task = CollectTaskModel::inst()->getTaskById($id);
if (empty($task)) {
$this->redirect[] = array('text' => '', 'href' => '/collect/task/index');
$this->message('采集规则不存在或已被删除', self::MSG_ERROR, true);
}
if ($_GET['type'] == 'list') {
$cacheKey = "jvod.collect.list.urls";
if ($this->redis->llen($cacheKey) == 0) {
$collect_list_urls = CollectTaskModel::getListUrls($task['collect_task_urls']);
foreach ($collect_list_urls as $_k => $_v) {
$this->redis->lpush($cacheKey, $_v);
}
}
for ($i = 0; $i < 20 && $this->redis->llen($cacheKey) > 0; $i++) {
$collect_list_url = $this->redis->rpop($cacheKey);
$collect_content_urls = CollectTaskModel::getContentUrls($collect_list_url, $task['collect_list_rules']['begin'], $task['collect_list_rules']['end']);
$repeat_number = 0;
$sql = "SELECT COUNT(collect_content_id) FROM {{collect_content}} WHERE collect_content_url=:collect_content_url";
foreach ($collect_content_urls as $_k => $_v) {
if ($this->db->queryScalar($sql, array(':collect_content_url' => $_v)) == 0) {
$this->db->insert('{{collect_content}}', array('collect_content_id' => 0, 'collect_task_id' => $id, 'collect_content_url' => $_v, 'is_published' => 0, 'is_collected' => 0, 'lasttime' => $_SERVER['REQUEST_TIME'], 'dateline' => $_SERVER['REQUEST_TIME']));
} else {
$repeat_number++;
}
if ($repeat_number > 20) {
// 超过20个重复地址，退出内容地址采集
break;
}
}
}
} else {
$cacheKey = "jvod.collect.content.urls";
if ($this->queue->llen($cacheKey) == 0) {
$sql = "SELECT collect_content_id, collect_task_id, collect_content_url, is_collected, is_published, lasttime, dateline FROM {{collect_content}} WHERE collect_task_id=:collect_task_id AND is_collected=:is_collected";
$ret = $this->db->queryAll($sql, array(':collect_task_id' => $id, ':is_collected' => 0));
foreach ($ret as $_k => $_v) {
$this->queue->lpush($cacheKey, json_encode(array($_v['collect_content_id'], $_v['collect_content_url'])));
}
}
for ($i = 0; $i < 20 && $this->queue->llen($cacheKey) > 0; $i++) {
list($collect_content_id, $collect_content_url) = json_decode($this->queue->rpop($cacheKey));
$collect_content_charset = 'utf-8';
$collect_content_body = CollectTask::get_url_contents($collect_content_url, $collect_content_charset);
if (strtolower($collect_content_charset) != 'utf-8') {
$collect_content_body = mb_convert_encoding($collect_content_body, 'UTF-8', $collect_content_charset);
}
$task['collect_content_data'] = array();
$data = array('collect_content_id' => $collect_content_id);
foreach ($task['collect_content_rules'] as $_k => $_v) {
if ($_v['begin'] && $_v['end']) {
preg_match("/{$_v['begin']}(.+?){$_v['end']}/s", $collect_content_body, $ret);
$data[$_v['collect_fields_identify']] = $ret[1] ? $ret[1] : '';
} else {
$data[$_v['collect_fields_identify']] = '';
}
}
$flag = $this->db->insert("{{collect_model_addons{$task['collect_model_identify']}}}", $data);
if ($flag) {
$this->db->update("{{collect_content}}", array('is_collected' => 1), 'collect_content_url=:collect_content_url', array(':collect_content_url' => $collect_content_url));
}
}
}
}
/**
*
* Enter description here ...
* @param unknown_type $params
*/
public function Pages($params = array())
{
//设置默认参数
$_defaults_params = array('allow_cache' => true, 'page' => isset($_GET['page']) ? intval($_GET['page']) : 1, 'pagesize' => 10);
$params = array_merge($_defaults_params, $params);
//有开启缓存功能，则从缓存中取数据， 如果有数据，则直接返回结果
if ($params['allow_cache'] && isset($this->cache)) {
$cacheKey = 'collect.template.pages.' . serialize($params);
$ret = $this->cache->get($cacheKey);
if ($_r && is_array($ret)) {
return $ret;
}
}
//添加条件
$builds = array('select' => 'COUNT(ct.collect_template_id) AS COUNT', 'from' => array('{{collect_template}}', 'ct'), 'join' => array(array('{{collect_source}}', 'cs', '`cs`.`collect_source_id`=`ct`.`collect_source_id`'), array('{{collect_model}}', 'cm', '`cm`.`collect_model_id`=`ct`.`collect_model_id`')));
$sql_params = array();
if (isset($params['collect_template_status']) && !empty($params['collect_template_status'])) {
$builds['where'][] = array('AND', 'ct.collect_template_status=:collect_template_status');
$sql_params[':collect_template_status'] = $params['collect_template_status'];
} else {
$builds['where'][] = array('AND', 'ct.collect_template_status>:collect_template_status');
$sql_params[':collect_template_status'] = 0;
}
//
if (isset($params['collect_template_id']) && !empty($params['collect_template_id'])) {
$builds['where'][] = array('AND', 'ct.collect_template_id=:collect_template_id');
$sql_params[':collect_template_id'] = $params['collect_template_id'];
}
if (isset($params['collect_model_id']) && !empty($params['collect_model_id'])) {
$builds['where'][] = array('AND', 'ct.collect_model_id=:collect_model_id');
$sql_params[':collect_model_id'] = $params['collect_model_id'];
}
if (isset($params['collect_source_id']) && !empty($params['collect_source_id'])) {
$builds['where'][] = array('AND', 'ct.collect_source_id=:collect_source_id');
$sql_params[':collect_source_id'] = $params['collect_source_id'];
}
if (isset($params['collect_template_id']) && !empty($params['collect_template_id'])) {
$builds['where'][] = array('AND', 'ct.collect_template_id=:collect_template_id');
$sql_params[':collect_template_id'] = $params['collect_template_id'];
}
//
if (isset($params['collect_template_name']) && !empty($params['collect_template_name'])) {
$builds['where'][] = array('LIKE', 'ct.collect_template_name', ':collect_template_name');
$sql_params[':collect_template_name'] = "%{$params['collect_template_name']}%";
}
//
//
if (isset($params['searchKey']) && $params['searchKey']) {
$builds['where'][] = array('LIKE', 'ct.collect_template_name', ':collect_template_name');
$sql_params[':collect_template_name'] = "%{$params['searchKey']}%";
}
$sql = $this->buildQuery($builds);
//统计数量
$count = $this->db->queryScalar($sql, $sql_params);
//分页处理
$pages = new CPagination($count);
//设置分页大小
$pages->pageSize = $params['pagesize'];
if (isset($params['orderby']) && $params['orderby']) {
$builds['order'] = $params['orderby'];
} else {
$builds['order'] = array('ct.collect_template_rank ASC', 'ct.collect_template_id DESC');
}
$builds['select'] = 'ct.collect_template_id,ct.collect_template_rank, ct.collect_template_name, ct.collect_template_charset, cs.collect_source_name, cm.collect_model_name, ct.collect_template_lasttime, ct.collect_template_dateline';
$pages->applyLimit($builds);
$sql = $this->buildQuery($builds);
$ret['pages'] = $pages;
$ret['rows'] = $this->db->queryAll($sql, $sql_params);
$charset = CollectTaskModel::getCharsets();
foreach ($ret['rows'] as $_k => $_v) {
$ret['rows'][$_k]['collect_template_charset'] = $charset[$_v['collect_template_charset']];
}
//有开启缓存，则把结果添加到缓存中
/*if($params['allow_cache'] && isset($this->cache)) {
$_cache_cache_time = Setting::get_setting_value('COLLECT_TEMPLATE_PAGES_CACHE_TIME');
$this->cache->set($cacheKey, json_encode($ret), $_cache_cache_time);
unset($_cache_cache_time, $cacheKey);
}*/
return $ret;
}