コード例 #1
0
ファイル: CollectTask.php プロジェクト: nbaiwan/yav
 /**
  * 强制更新缓存
  */
 public function updateCache()
 {
     //
     self::$__tasks = null;
     //
     self::$__tasks = $this->readTaskArray();
     if (isset($this->cache)) {
         $this->cache->set('collect.task', $__tasks);
     }
     return true;
 }
コード例 #2
0
ファイル: Template.php プロジェクト: nbaiwan/yav
 public function loadCharsetAction($id)
 {
     $template = CollectTemplateModel::getTemplateById($id, false);
     $_charset = CollectTaskModel::getCharset();
     echo $_charset[$template['collect_template_charset']];
 }
コード例 #3
0
ファイル: Task.php プロジェクト: nbaiwan/yav
 public function collectAction($id)
 {
     $task = CollectTaskModel::inst()->getTaskById($id);
     if (empty($task)) {
         $this->redirect[] = array('text' => '', 'href' => '/collect/task/index');
         $this->message('采集规则不存在或已被删除', self::MSG_ERROR, true);
     }
     if ($_GET['type'] == 'list') {
         $cacheKey = "jvod.collect.list.urls";
         if ($this->redis->llen($cacheKey) == 0) {
             $collect_list_urls = CollectTaskModel::getListUrls($task['collect_task_urls']);
             foreach ($collect_list_urls as $_k => $_v) {
                 $this->redis->lpush($cacheKey, $_v);
             }
         }
         for ($i = 0; $i < 20 && $this->redis->llen($cacheKey) > 0; $i++) {
             $collect_list_url = $this->redis->rpop($cacheKey);
             $collect_content_urls = CollectTaskModel::getContentUrls($collect_list_url, $task['collect_list_rules']['begin'], $task['collect_list_rules']['end']);
             $repeat_number = 0;
             $sql = "SELECT COUNT(collect_content_id) FROM {{collect_content}} WHERE collect_content_url=:collect_content_url";
             foreach ($collect_content_urls as $_k => $_v) {
                 if ($this->db->queryScalar($sql, array(':collect_content_url' => $_v)) == 0) {
                     $this->db->insert('{{collect_content}}', array('collect_content_id' => 0, 'collect_task_id' => $id, 'collect_content_url' => $_v, 'is_published' => 0, 'is_collected' => 0, 'lasttime' => $_SERVER['REQUEST_TIME'], 'dateline' => $_SERVER['REQUEST_TIME']));
                 } else {
                     $repeat_number++;
                 }
                 if ($repeat_number > 20) {
                     // 超过20个重复地址,退出内容地址采集
                     break;
                 }
             }
         }
     } else {
         $cacheKey = "jvod.collect.content.urls";
         if ($this->queue->llen($cacheKey) == 0) {
             $sql = "SELECT collect_content_id, collect_task_id, collect_content_url, is_collected, is_published, lasttime, dateline FROM {{collect_content}} WHERE collect_task_id=:collect_task_id AND is_collected=:is_collected";
             $ret = $this->db->queryAll($sql, array(':collect_task_id' => $id, ':is_collected' => 0));
             foreach ($ret as $_k => $_v) {
                 $this->queue->lpush($cacheKey, json_encode(array($_v['collect_content_id'], $_v['collect_content_url'])));
             }
         }
         for ($i = 0; $i < 20 && $this->queue->llen($cacheKey) > 0; $i++) {
             list($collect_content_id, $collect_content_url) = json_decode($this->queue->rpop($cacheKey));
             $collect_content_charset = 'utf-8';
             $collect_content_body = CollectTask::get_url_contents($collect_content_url, $collect_content_charset);
             if (strtolower($collect_content_charset) != 'utf-8') {
                 $collect_content_body = mb_convert_encoding($collect_content_body, 'UTF-8', $collect_content_charset);
             }
             $task['collect_content_data'] = array();
             $data = array('collect_content_id' => $collect_content_id);
             foreach ($task['collect_content_rules'] as $_k => $_v) {
                 if ($_v['begin'] && $_v['end']) {
                     preg_match("/{$_v['begin']}(.+?){$_v['end']}/s", $collect_content_body, $ret);
                     $data[$_v['collect_fields_identify']] = $ret[1] ? $ret[1] : '';
                 } else {
                     $data[$_v['collect_fields_identify']] = '';
                 }
             }
             $flag = $this->db->insert("{{collect_model_addons{$task['collect_model_identify']}}}", $data);
             if ($flag) {
                 $this->db->update("{{collect_content}}", array('is_collected' => 1), 'collect_content_url=:collect_content_url', array(':collect_content_url' => $collect_content_url));
             }
         }
     }
 }
コード例 #4
0
ファイル: CollectTemplate.php プロジェクト: nbaiwan/yav
 /**
  * 
  * Enter description here ...
  * @param unknown_type $params
  */
 public function Pages($params = array())
 {
     //设置默认参数
     $_defaults_params = array('allow_cache' => true, 'page' => isset($_GET['page']) ? intval($_GET['page']) : 1, 'pagesize' => 10);
     $params = array_merge($_defaults_params, $params);
     //有开启缓存功能,则从缓存中取数据, 如果有数据,则直接返回结果
     if ($params['allow_cache'] && isset($this->cache)) {
         $cacheKey = 'collect.template.pages.' . serialize($params);
         $ret = $this->cache->get($cacheKey);
         if ($_r && is_array($ret)) {
             return $ret;
         }
     }
     //添加条件
     $builds = array('select' => 'COUNT(ct.collect_template_id) AS COUNT', 'from' => array('{{collect_template}}', 'ct'), 'join' => array(array('{{collect_source}}', 'cs', '`cs`.`collect_source_id`=`ct`.`collect_source_id`'), array('{{collect_model}}', 'cm', '`cm`.`collect_model_id`=`ct`.`collect_model_id`')));
     $sql_params = array();
     if (isset($params['collect_template_status']) && !empty($params['collect_template_status'])) {
         $builds['where'][] = array('AND', 'ct.collect_template_status=:collect_template_status');
         $sql_params[':collect_template_status'] = $params['collect_template_status'];
     } else {
         $builds['where'][] = array('AND', 'ct.collect_template_status>:collect_template_status');
         $sql_params[':collect_template_status'] = 0;
     }
     //
     if (isset($params['collect_template_id']) && !empty($params['collect_template_id'])) {
         $builds['where'][] = array('AND', 'ct.collect_template_id=:collect_template_id');
         $sql_params[':collect_template_id'] = $params['collect_template_id'];
     }
     if (isset($params['collect_model_id']) && !empty($params['collect_model_id'])) {
         $builds['where'][] = array('AND', 'ct.collect_model_id=:collect_model_id');
         $sql_params[':collect_model_id'] = $params['collect_model_id'];
     }
     if (isset($params['collect_source_id']) && !empty($params['collect_source_id'])) {
         $builds['where'][] = array('AND', 'ct.collect_source_id=:collect_source_id');
         $sql_params[':collect_source_id'] = $params['collect_source_id'];
     }
     if (isset($params['collect_template_id']) && !empty($params['collect_template_id'])) {
         $builds['where'][] = array('AND', 'ct.collect_template_id=:collect_template_id');
         $sql_params[':collect_template_id'] = $params['collect_template_id'];
     }
     //
     if (isset($params['collect_template_name']) && !empty($params['collect_template_name'])) {
         $builds['where'][] = array('LIKE', 'ct.collect_template_name', ':collect_template_name');
         $sql_params[':collect_template_name'] = "%{$params['collect_template_name']}%";
     }
     //
     //
     if (isset($params['searchKey']) && $params['searchKey']) {
         $builds['where'][] = array('LIKE', 'ct.collect_template_name', ':collect_template_name');
         $sql_params[':collect_template_name'] = "%{$params['searchKey']}%";
     }
     $sql = $this->buildQuery($builds);
     //统计数量
     $count = $this->db->queryScalar($sql, $sql_params);
     //分页处理
     $pages = new CPagination($count);
     //设置分页大小
     $pages->pageSize = $params['pagesize'];
     if (isset($params['orderby']) && $params['orderby']) {
         $builds['order'] = $params['orderby'];
     } else {
         $builds['order'] = array('ct.collect_template_rank ASC', 'ct.collect_template_id DESC');
     }
     $builds['select'] = 'ct.collect_template_id,ct.collect_template_rank, ct.collect_template_name, ct.collect_template_charset, cs.collect_source_name, cm.collect_model_name, ct.collect_template_lasttime, ct.collect_template_dateline';
     $pages->applyLimit($builds);
     $sql = $this->buildQuery($builds);
     $ret['pages'] = $pages;
     $ret['rows'] = $this->db->queryAll($sql, $sql_params);
     $charset = CollectTaskModel::getCharsets();
     foreach ($ret['rows'] as $_k => $_v) {
         $ret['rows'][$_k]['collect_template_charset'] = $charset[$_v['collect_template_charset']];
     }
     //有开启缓存,则把结果添加到缓存中
     /*if($params['allow_cache'] && isset($this->cache)) {
     			$_cache_cache_time = Setting::get_setting_value('COLLECT_TEMPLATE_PAGES_CACHE_TIME');
     			$this->cache->set($cacheKey, json_encode($ret), $_cache_cache_time);
     			unset($_cache_cache_time, $cacheKey);
     		}*/
     return $ret;
 }