/** * Generate the module */ protected function compile() { /** @var PageModel $objPage */ global $objPage; if ($this->rss_template != 'rss_default') { $this->strTemplate = $this->rss_template; /** @var FrontendTemplate|object $objTemplate */ $objTemplate = new \FrontendTemplate($this->strTemplate); $this->Template = $objTemplate; $this->Template->setData($this->arrData); } $this->Template->link = $this->objFeed->get_link(); $this->Template->title = $this->objFeed->get_title(); $this->Template->language = $this->objFeed->get_language(); $this->Template->description = $this->objFeed->get_description(); $this->Template->copyright = $this->objFeed->get_copyright(); // Add image if ($this->objFeed->get_image_url()) { $this->Template->image = true; $this->Template->src = $this->objFeed->get_image_url(); $this->Template->alt = $this->objFeed->get_image_title(); $this->Template->href = $this->objFeed->get_image_link(); $this->Template->height = $this->objFeed->get_image_height(); $this->Template->width = $this->objFeed->get_image_width(); } // Get the items (see #6107) $arrItems = array_slice($this->objFeed->get_items(0, intval($this->numberOfItems) + intval($this->skipFirst)), intval($this->skipFirst), intval($this->numberOfItems) ?: null); $limit = count($arrItems); $offset = 0; // Split pages if ($this->perPage > 0) { // Get the current page $id = 'page_r' . $this->id; $page = \Input::get($id) !== null ? \Input::get($id) : 1; // Do not index or cache the page if the page number is outside the range if ($page < 1 || $page > max(ceil(count($arrItems) / $this->perPage), 1)) { throw new PageNotFoundException('Page not found: ' . \Environment::get('uri')); } // Set limit and offset $offset = ($page - 1) * $this->perPage; $limit = $this->perPage + $offset; $objPagination = new \Pagination(count($arrItems), $this->perPage, \Config::get('maxPaginationLinks'), $id); $this->Template->pagination = $objPagination->generate("\n "); } $items = array(); $last = min($limit, count($arrItems)) - 1; /** @var \SimplePie_Item[] $arrItems */ for ($i = $offset, $c = count($arrItems); $i < $limit && $i < $c; $i++) { $items[$i] = array('link' => $arrItems[$i]->get_link(), 'title' => $arrItems[$i]->get_title(), 'permalink' => $arrItems[$i]->get_permalink(), 'description' => str_replace(array('<?', '?>'), array('<?', '?>'), $arrItems[$i]->get_description()), 'class' => ($i == 0 ? ' first' : '') . ($i == $last ? ' last' : '') . ($i % 2 == 0 ? ' even' : ' odd'), 'pubdate' => \Date::parse($objPage->datimFormat, $arrItems[$i]->get_date('U')), 'category' => $arrItems[$i]->get_category(0), 'object' => $arrItems[$i]); // Add author if (($objAuthor = $arrItems[$i]->get_author(0)) != false) { $items[$i]['author'] = trim($objAuthor->name . ' ' . $objAuthor->email); } // Add enclosure if (($objEnclosure = $arrItems[$i]->get_enclosure(0)) != false) { $items[$i]['enclosure'] = $objEnclosure->get_link(); } } $this->Template->items = array_values($items); }
/** * Add a new feed to the database * * Adds the specified feed name and URL to the database. If no name is set * by the user, it fetches one from the feed. If the URL specified is a HTML * page and not a feed, it lets SimplePie do autodiscovery and uses the XML * url returned. * * @since 1.0 * * @param string $url URL to feed or website (if autodiscovering) * @param string $name Title/Name of feed * @param string $cat Category to add feed to * @return bool True if succeeded, false if failed */ public function add($url, $name = '', $cat = 'default') { if (empty($url)) { throw new Exception(_r("Couldn't add feed: No feed URL supplied"), Errors::get_code('admin.feeds.no_url')); } if (!preg_match('#https|http|feed#', $url)) { if (strpos($url, '://')) { throw new Exception(_r('Unsupported URL protocol'), Errors::get_code('admin.feeds.protocol_error')); } $url = 'http://' . $url; } require_once LILINA_INCPATH . '/contrib/simplepie/simplepie.inc'; $feed_info = new SimplePie(); $feed_info->set_useragent(LILINA_USERAGENT . ' SimplePie/' . SIMPLEPIE_BUILD); $feed_info->set_stupidly_fast(true); $feed_info->set_cache_location(get_option('cachedir')); $feed_info->set_favicon_handler(get_option('baseurl') . '/lilina-favicon.php'); $feed_info->set_feed_url($url); $feed_info->init(); $feed_error = $feed_info->error(); $feed_url = $feed_info->subscribe_url(); if (!empty($feed_error)) { throw new Exception(sprintf(_r("Couldn't add feed: %s is not a valid URL or the server could not be accessed. Additionally, no feeds could be found by autodiscovery."), $url), Errors::get_code('admin.feeds.invalid_url')); } if (empty($name)) { //Get it from the feed $name = $feed_info->get_title(); } $id = sha1($feed_url); $this->feeds[$id] = array('feed' => $feed_url, 'url' => $feed_info->get_link(), 'id' => $id, 'name' => $name, 'cat' => $cat, 'icon' => $feed_info->get_favicon()); $this->feeds[$id] = apply_filters('feed-create', $this->feeds[$id], $url); $this->save(); return array('msg' => sprintf(_r('Added feed "%1$s"'), $name), 'id' => $id); }
static function add($url) { require_once 'lib/simplepie_1.3.compiled.php'; $feed = new SimplePie(); $feed->set_feed_url($_POST['url']); $feed->enable_cache(false); $feed->init(); $feed->handle_content_type(); $values = array(':id' => $feed->feed_url, ':site' => $feed->get_link(), ':title' => $feed->get_title()); $result = Feeds::$db->exec("INSERT OR IGNORE INTO feeds " . "(id, site_url, title) VALUES (:id, :site, :title)", $values); return $result == 0 ? FALSE : TRUE; }
/** * Implements CollectionAbstract::buildDocument(). * * @param IndexDocument $document * @param \SimplePie_Item $data */ public function buildDocument(IndexDocument $document, $data) { $document->source = $this->_feed->get_title(); $document->subject = $this->_feed->get_description(); $document->title = $data->get_title(); $document->link = $data->get_link(); $document->description = $data->get_description(); $document->creator = (array) $data->get_author(); $document->date = $data->get_date(); // PHP properties cannot have dashes (-), and the fields below have // dashes in the field name. $document->source_link = $this->_feed->get_link(); $document->getField('source_link')->setName('source-link'); $document->item_subject = $this->_feed->get_link(); $document->getField('item_subject')->setName('item-subject'); }
/** * Find the feed's icon * * @param SimplePie $feed SimplePie object to retrieve logo for * @return string URL to feed icon */ protected static function discover_favicon($feed, $id) { if ($return = $feed->get_channel_tags(SIMPLEPIE_NAMESPACE_ATOM_10, 'icon')) { $favicon = SimplePie_Misc::absolutize_url($return[0]['data'], $feed->get_base($return[0])); } elseif (($url = $feed->get_link()) !== null && preg_match('/^http(s)?:\\/\\//i', $url)) { $filename = $id . '.ico'; $favicon = SimplePie_Misc::absolutize_url('/favicon.ico', $url); } else { return false; } $cache = new DataHandler(get_option('cachedir')); $request = new HTTPRequest(); $file = $request->get($favicon, array('X-Forwarded-For' => $_SERVER['REMOTE_ADDR'])); if ($file->success && strlen($file->body) > 0) { $sniffer = new $feed->content_type_sniffer_class($file); if (substr($sniffer->get_type(), 0, 6) === 'image/') { $body = array('type' => $sniffer->get_type(), 'body' => $file->body); return $cache->save($filename, serialize($body)); } else { return false; } } return false; }
public function items() { if ($this->input->is_ajax_request()) { $this->readerself_library->set_template('_json'); $this->readerself_library->set_content_type('application/json'); $content = array(); } else { $this->readerself_library->set_template('_plain'); $this->readerself_library->set_content_type('text/plain'); $content = ''; } if ($this->input->is_cli_request() && !$this->config->item('refresh_by_cron')) { $content .= 'Refresh by cron disabled' . "\n"; } else { include_once 'thirdparty/simplepie/autoloader.php'; include_once 'thirdparty/simplepie/idn/idna_convert.class.php'; if ($this->config->item('facebook/enabled')) { include_once 'thirdparty/facebook/autoload.php'; $fb = new Facebook\Facebook(array('app_id' => $this->config->item('facebook/id'), 'app_secret' => $this->config->item('facebook/secret'))); $fbApp = $fb->getApp(); $accessToken = $fbApp->getAccessToken(); } $query = $this->db->query('SELECT fed.* FROM ' . $this->db->dbprefix('feeds') . ' AS fed WHERE fed.fed_nextcrawl IS NULL OR fed.fed_nextcrawl <= ? GROUP BY fed.fed_id HAVING (SELECT COUNT(DISTINCT(sub.mbr_id)) FROM ' . $this->db->dbprefix('subscriptions') . ' AS sub WHERE sub.fed_id = fed.fed_id) > 0', array(date('Y-m-d H:i:s'))); if ($query->num_rows() > 0) { $microtime_start = microtime(1); $errors = 0; foreach ($query->result() as $fed) { $parse_url = parse_url($fed->fed_link); if (isset($parse_url['host']) == 1 && $parse_url['host'] == 'www.facebook.com' && $this->config->item('facebook/enabled')) { try { $parts = explode('/', $parse_url['path']); $total_parts = count($parts); $last_part = $parts[$total_parts - 1]; $request = new Facebook\FacebookRequest($fbApp, $accessToken, 'GET', $last_part . '?fields=link,name,about'); $response = $fb->getClient()->sendRequest($request); $result = $response->getDecodedBody(); $request = new Facebook\FacebookRequest($fbApp, $accessToken, 'GET', $last_part . '?fields=feed{created_time,id,message,story,full_picture,place,type,status_type,link}'); $response = $fb->getClient()->sendRequest($request); $posts = $response->getDecodedBody(); $this->readerself_library->crawl_items_facebook($fed->fed_id, $posts['feed']['data']); $lastitem = $this->db->query('SELECT itm.itm_datecreated FROM ' . $this->db->dbprefix('items') . ' AS itm WHERE itm.fed_id = ? GROUP BY itm.itm_id ORDER BY itm.itm_id DESC LIMIT 0,1', array($fed->fed_id))->row(); $this->db->set('fed_title', $result['name']); $this->db->set('fed_url', $result['link']); $this->db->set('fed_link', $result['link']); if (isset($parse_url['host']) == 1) { $this->db->set('fed_host', $parse_url['host']); } $this->db->set('fed_description', $result['about']); $this->db->set('fed_lasterror', ''); $this->db->set('fed_lastcrawl', date('Y-m-d H:i:s')); if ($lastitem) { $nextcrawl = ''; //older than 96 hours, next crawl in 12 hours if ($lastitem->itm_datecreated < date('Y-m-d H:i:s', time() - 3600 * 24 * 96)) { $nextcrawl = date('Y-m-d H:i:s', time() + 3600 * 12); //older than 48 hours, next crawl in 6 hours } else { if ($lastitem->itm_datecreated < date('Y-m-d H:i:s', time() - 3600 * 48)) { $nextcrawl = date('Y-m-d H:i:s', time() + 3600 * 6); //older than 24 hours, next crawl in 3 hours } else { if ($lastitem->itm_datecreated < date('Y-m-d H:i:s', time() - 3600 * 24)) { $nextcrawl = date('Y-m-d H:i:s', time() + 3600 * 3); } } } $this->db->set('fed_nextcrawl', $nextcrawl); } $this->db->where('fed_id', $fed->fed_id); $this->db->update('feeds'); } catch (Facebook\Exceptions\FacebookResponseException $e) { $errors++; $this->db->set('fed_lasterror', 'Graph returned an error: ' . $e->getMessage()); $this->db->set('fed_lastcrawl', date('Y-m-d H:i:s')); $this->db->where('fed_id', $fed->fed_id); $this->db->update('feeds'); } catch (Facebook\Exceptions\FacebookSDKException $e) { $errors++; $this->db->set('fed_lasterror', 'Facebook SDK returned an error: ' . $e->getMessage()); $this->db->set('fed_lastcrawl', date('Y-m-d H:i:s')); $this->db->where('fed_id', $fed->fed_id); $this->db->update('feeds'); } } else { $sp_feed = new SimplePie(); $sp_feed->set_feed_url(convert_to_ascii($fed->fed_link)); $sp_feed->enable_cache(false); $sp_feed->set_timeout(5); $sp_feed->force_feed(true); $sp_feed->init(); $sp_feed->handle_content_type(); if ($sp_feed->error()) { $errors++; $this->db->set('fed_lasterror', $sp_feed->error()); $this->db->set('fed_lastcrawl', date('Y-m-d H:i:s')); $this->db->where('fed_id', $fed->fed_id); $this->db->update('feeds'); } else { $this->readerself_library->crawl_items($fed->fed_id, $sp_feed->get_items()); $lastitem = $this->db->query('SELECT itm.itm_datecreated FROM ' . $this->db->dbprefix('items') . ' AS itm WHERE itm.fed_id = ? GROUP BY itm.itm_id ORDER BY itm.itm_id DESC LIMIT 0,1', array($fed->fed_id))->row(); $parse_url = parse_url($sp_feed->get_link()); $this->db->set('fed_title', $sp_feed->get_title()); $this->db->set('fed_url', $sp_feed->get_link()); $this->db->set('fed_link', $sp_feed->subscribe_url()); if (isset($parse_url['host']) == 1) { $this->db->set('fed_host', $parse_url['host']); } if ($sp_feed->get_type() & SIMPLEPIE_TYPE_RSS_ALL) { $this->db->set('fed_type', 'rss'); } else { if ($sp_feed->get_type() & SIMPLEPIE_TYPE_ATOM_ALL) { $this->db->set('fed_type', 'atom'); } } if ($sp_feed->get_image_url()) { $this->db->set('fed_image', $sp_feed->get_image_url()); } $this->db->set('fed_description', $sp_feed->get_description()); $this->db->set('fed_lasterror', ''); $this->db->set('fed_lastcrawl', date('Y-m-d H:i:s')); if ($lastitem) { $nextcrawl = ''; //older than 96 hours, next crawl in 12 hours if ($lastitem->itm_datecreated < date('Y-m-d H:i:s', time() - 3600 * 24 * 96)) { $nextcrawl = date('Y-m-d H:i:s', time() + 3600 * 12); //older than 48 hours, next crawl in 6 hours } else { if ($lastitem->itm_datecreated < date('Y-m-d H:i:s', time() - 3600 * 48)) { $nextcrawl = date('Y-m-d H:i:s', time() + 3600 * 6); //older than 24 hours, next crawl in 3 hours } else { if ($lastitem->itm_datecreated < date('Y-m-d H:i:s', time() - 3600 * 24)) { $nextcrawl = date('Y-m-d H:i:s', time() + 3600 * 3); } } } $this->db->set('fed_nextcrawl', $nextcrawl); } $this->db->where('fed_id', $fed->fed_id); $this->db->update('feeds'); } $sp_feed->__destruct(); unset($sp_feed); } } $this->db->set('crr_time', microtime(1) - $microtime_start); if (function_exists('memory_get_peak_usage')) { $this->db->set('crr_memory', memory_get_peak_usage()); } $this->db->set('crr_feeds', $query->num_rows()); if ($errors > 0) { $this->db->set('crr_errors', $errors); } $this->db->set('crr_datecreated', date('Y-m-d H:i:s')); $this->db->insert('crawler'); if ($this->db->dbdriver == 'mysqli') { $this->db->query('OPTIMIZE TABLE categories, connections, enclosures, favorites, feeds, folders, history, items, members, share, subscriptions'); } } } $this->readerself_library->set_content($content); }
public function create() { if (!$this->axipi_session->userdata('mbr_id')) { redirect(base_url() . '?u=' . $this->input->get('u')); } $data = array(); $content = array(); $this->load->library(array('form_validation', 'analyzer_library')); if ($this->config->item('folders')) { $query = $this->db->query('SELECT flr.* FROM ' . $this->db->dbprefix('folders') . ' AS flr WHERE flr.mbr_id = ? GROUP BY flr.flr_id ORDER BY flr.flr_title ASC', array($this->member->mbr_id)); $data['folders'] = array(); $data['folders'][0] = $this->lang->line('no_folder'); if ($query->num_rows() > 0) { foreach ($query->result() as $flr) { $data['folders'][$flr->flr_id] = $flr->flr_title; } } } $this->form_validation->set_rules('url', 'lang:url_feed', 'required'); if ($this->config->item('folders')) { $this->form_validation->set_rules('folder', 'lang:folder', 'required'); } $this->form_validation->set_rules('priority', 'lang:priority', 'numeric'); //$this->form_validation->set_rules('direction', 'lang:direction', ''); $data['error'] = false; $data['feeds'] = array(); if ($this->input->post('url') && !$this->input->post('analyze_done')) { $this->analyzer_library->start($this->input->post('url')); $metas = $this->analyzer_library->metas; if (count($metas) > 0) { $data['feeds'][''] = '-'; foreach ($metas as $meta) { $add = true; $headers = get_headers($meta['href'], 1); if (isset($headers['Location']) == 1) { $meta['href'] = $headers['Location']; $headers = get_headers($meta['href'], 1); if (isset($headers['Location']) == 1) { $add = false; } } if ($add) { if ($meta['title'] == '') { $data['feeds'][$meta['href']] = $meta['href']; } else { $this->analyzer_library->encoding($meta['title']); $data['feeds'][$meta['href']] = $meta['title']; } } } } } if ($this->form_validation->run() == FALSE || count($data['feeds']) > 0) { $content = $this->load->view('subscriptions_create', $data, TRUE); } else { if ($this->config->item('folders')) { $folder = false; if ($this->input->post('folder')) { $query = $this->db->query('SELECT flr.* FROM ' . $this->db->dbprefix('folders') . ' AS flr WHERE flr.mbr_id = ? AND flr.flr_id = ? GROUP BY flr.flr_id', array($this->member->mbr_id, $this->input->post('folder'))); if ($query->num_rows() > 0) { $folder = $this->input->post('folder'); } } } $query = $this->db->query('SELECT fed.*, sub.sub_id FROM ' . $this->db->dbprefix('feeds') . ' AS fed LEFT JOIN ' . $this->db->dbprefix('subscriptions') . ' AS sub ON sub.fed_id = fed.fed_id AND sub.mbr_id = ? WHERE fed.fed_link = ? GROUP BY fed.fed_id', array($this->member->mbr_id, $this->input->post('url'))); if ($query->num_rows() == 0) { $parse_url = parse_url($this->input->post('url')); if (isset($parse_url['host']) == 1 && $parse_url['host'] == 'instagram.com' && $this->config->item('instagram/enabled')) { if ($this->config->item('instagram/access_token')) { $parts = explode('/', rtrim($parse_url['path'], '/')); $total_parts = count($parts); $last_part = $parts[$total_parts - 1]; $result = json_decode(file_get_contents('https://api.instagram.com/v1/users/search?q=' . $last_part . '&count=15&access_token=' . $this->config->item('instagram/access_token'))); if (count($result->data) == 0) { $data['error'] = 'User not found'; } else { $user_id = false; foreach ($result->data as $user) { if ($user->username == $last_part) { $user_id = $user->id; break; } } if (!$user_id) { $data['error'] = 'User not found'; } else { $result = json_decode(file_get_contents('https://api.instagram.com/v1/users/' . $user_id . '?access_token=' . $this->config->item('instagram/access_token'))); $this->db->set('fed_title', 'Instagram @' . $result->data->username); $this->db->set('fed_url', $this->input->post('url')); $this->db->set('fed_description', $result->data->bio); $this->db->set('fed_image', $result->data->profile_picture); $this->db->set('fed_link', $this->input->post('url')); if (isset($parse_url['host']) == 1) { $this->db->set('fed_host', $parse_url['host']); } $this->db->set('fed_lastcrawl', date('Y-m-d H:i:s')); $this->db->set('fed_datecreated', date('Y-m-d H:i:s')); $this->db->insert('feeds'); $fed_id = $this->db->insert_id(); $this->db->set('mbr_id', $this->member->mbr_id); $this->db->set('fed_id', $fed_id); if ($this->config->item('folders')) { if ($folder) { $this->db->set('flr_id', $folder); } } $this->db->set('sub_priority', $this->input->post('priority')); $this->db->set('sub_direction', $this->input->post('direction')); $this->db->set('sub_datecreated', date('Y-m-d H:i:s')); $this->db->insert('subscriptions'); $sub_id = $this->db->insert_id(); $result = json_decode(file_get_contents('https://api.instagram.com/v1/users/' . $user_id . '/media/recent?access_token=' . $this->config->item('instagram/access_token'))); $this->readerself_library->crawl_items_instagram($fed_id, $result->data); } } } } else { if (isset($parse_url['host']) == 1 && $parse_url['host'] == 'www.facebook.com' && $this->config->item('facebook/enabled')) { include_once 'thirdparty/facebook/autoload.php'; $fb = new Facebook\Facebook(array('app_id' => $this->config->item('facebook/id'), 'app_secret' => $this->config->item('facebook/secret'))); $fbApp = $fb->getApp(); $accessToken = $fbApp->getAccessToken(); try { $parts = explode('/', rtrim($parse_url['path'], '/')); $total_parts = count($parts); $last_part = $parts[$total_parts - 1]; $request = new Facebook\FacebookRequest($fbApp, $accessToken, 'GET', $last_part . '?fields=link,name,about'); $response = $fb->getClient()->sendRequest($request); $result = $response->getDecodedBody(); $this->db->set('fed_title', $result['name']); $this->db->set('fed_url', $result['link']); $this->db->set('fed_description', $result['about']); $this->db->set('fed_link', $result['link']); if (isset($parse_url['host']) == 1) { $this->db->set('fed_host', $parse_url['host']); } $this->db->set('fed_lastcrawl', date('Y-m-d H:i:s')); $this->db->set('fed_datecreated', date('Y-m-d H:i:s')); $this->db->insert('feeds'); $fed_id = $this->db->insert_id(); $this->db->set('mbr_id', $this->member->mbr_id); $this->db->set('fed_id', $fed_id); if ($this->config->item('folders')) { if ($folder) { $this->db->set('flr_id', $folder); } } $this->db->set('sub_priority', $this->input->post('priority')); $this->db->set('sub_direction', $this->input->post('direction')); $this->db->set('sub_datecreated', date('Y-m-d H:i:s')); $this->db->insert('subscriptions'); $sub_id = $this->db->insert_id(); $request = new Facebook\FacebookRequest($fbApp, $accessToken, 'GET', $last_part . '?fields=feed{created_time,id,message,story,full_picture,place,type,status_type,link}'); $response = $fb->getClient()->sendRequest($request); $posts = $response->getDecodedBody(); $this->readerself_library->crawl_items_facebook($fed_id, $posts['feed']['data']); redirect(base_url() . 'subscriptions/read/' . $sub_id); } catch (Facebook\Exceptions\FacebookResponseException $e) { $data['error'] = 'Graph returned an error: ' . $e->getMessage(); } catch (Facebook\Exceptions\FacebookSDKException $e) { $data['error'] = 'Facebook SDK returned an error: ' . $e->getMessage(); } } else { include_once 'thirdparty/simplepie/autoloader.php'; include_once 'thirdparty/simplepie/idn/idna_convert.class.php'; $sp_feed = new SimplePie(); $sp_feed->set_feed_url(convert_to_ascii($this->input->post('url'))); $sp_feed->enable_cache(false); $sp_feed->set_timeout(60); $sp_feed->force_feed(true); $sp_feed->init(); $sp_feed->handle_content_type(); if ($sp_feed->error()) { $data['error'] = $sp_feed->error(); } else { $parse_url = parse_url($sp_feed->get_link()); $this->db->set('fed_title', $sp_feed->get_title()); $this->db->set('fed_url', $sp_feed->get_link()); $this->db->set('fed_description', $sp_feed->get_description()); $this->db->set('fed_link', $sp_feed->subscribe_url()); if (isset($parse_url['host']) == 1) { $this->db->set('fed_host', $parse_url['host']); } $this->db->set('fed_lastcrawl', date('Y-m-d H:i:s')); $this->db->set('fed_datecreated', date('Y-m-d H:i:s')); $this->db->insert('feeds'); $fed_id = $this->db->insert_id(); $this->db->set('mbr_id', $this->member->mbr_id); $this->db->set('fed_id', $fed_id); if ($this->config->item('folders')) { if ($folder) { $this->db->set('flr_id', $folder); } } $this->db->set('sub_priority', $this->input->post('priority')); $this->db->set('sub_direction', $this->input->post('direction')); $this->db->set('sub_datecreated', date('Y-m-d H:i:s')); $this->db->insert('subscriptions'); $sub_id = $this->db->insert_id(); $data['sub_id'] = $sub_id; $data['fed_title'] = $sp_feed->get_title(); $this->readerself_library->crawl_items($fed_id, $sp_feed->get_items()); } $sp_feed->__destruct(); unset($sp_feed); } } } else { $fed = $query->row(); if (!$fed->sub_id) { $this->db->set('mbr_id', $this->member->mbr_id); $this->db->set('fed_id', $fed->fed_id); if ($this->config->item('folders')) { if ($folder) { $this->db->set('flr_id', $folder); } } $this->db->set('sub_priority', $this->input->post('priority')); $this->db->set('sub_direction', $this->input->post('direction')); $this->db->set('sub_datecreated', date('Y-m-d H:i:s')); $this->db->insert('subscriptions'); $sub_id = $this->db->insert_id(); } else { $sub_id = $fed->sub_id; } $data['sub_id'] = $sub_id; $data['fed_title'] = $fed->fed_title; } if ($data['error']) { $content = $this->load->view('subscriptions_create', $data, TRUE); } else { redirect(base_url() . 'subscriptions/read/' . $sub_id); } } $this->readerself_library->set_content($content); }
/** * get title, url and preview for rss-feed * * @param string $feed_url * @param int $max_items maximum number of items to fetch * @return array * @access */ function getInfoFromFeed($username, $service_type_id, $feed_url, $max_items = 5) { # needed for autodiscovery of feed //require 'simplepie/simplepie.php'; $feed = new SimplePie(); $feed->set_cache_location(CACHE . 'simplepie'); $feed->set_feed_url($feed_url); $feed->set_autodiscovery_level(SIMPLEPIE_LOCATOR_ALL); @$feed->init(); if ($feed->error()) { return false; } $data = array(); $data['title'] = $feed->get_title(); $data['account_url'] = $feed->get_link(); $data['feed_url'] = $feed->feed_url; unset($feed); if (!$data['account_url']) { $data['account_url'] = $data['feed_url']; } $data['service_id'] = 8; # any RSS-Feed $data['username'] = '******'; $data['service_type_id'] = $service_type_id; $items = $this->feed2array($username, 8, $data['service_type_id'], $data['feed_url'], 5, null); if (!$items) { return false; } $data['items'] = $items; return $data; }
function fof_update_feed($url) { global $FOF_FEED_TABLE; # # Get feed data. # if (!$url) { return 0; } if (!empty($url)) { $piefeed = new SimplePie(); $piefeed->set_feed_url($url); } else { return 0; } $piefeed->set_cache_location(FOF_CACHE_DIR); // echo "Memory usage after set cache: " . number_format(memory_get_usage()); $piefeed->init(); //echo "Memory usage after set init: " . number_format(memory_get_usage()); $piefeed->handle_content_type(); //echo "Memory usage after contenttype: " . number_format(memory_get_usage()); if (!$piefeed->data) { //print "<font color=\"red\">" . _("error was") . "</font>: <B>" . $piefeed->error . "</b> "; //print "<a href=\"http://feedvalidator.org/check?url=$url\">" . _("try to validate it?") . "</a> "; //unset($piefeed); $piefeed->__destruct(); // Do what PHP should be doing on it's own. return 0; } $title = $piefeed->get_title(); $link = $piefeed->get_link(); $description = $piefeed->get_description(); $safeurl = mysql_escape_string($url); $result = fof_do_query("select id, url, aging from feeds where url='{$safeurl}'"); $row = mysql_fetch_array($result); mysql_free_result($result); // echo "Memory usage after first query: " . number_format(memory_get_usage()); $feed_id = $row['id']; $keep_days = $row['aging']; if ($keep_days < 0) { $keep_days = 60; } // $result2 = fof_do_query("select image from feeds where `id`='$feed_id'"); // $row2 = mysql_fetch_array($result2); // mysql_free_result($result); //echo "Memory usage after second query: " . number_format(memory_get_usage()); // $image2 = $row2['image']; /* if (!$image2) { $imagelink = $piefeed->get_favicon(true, ''); $HTTPRequest = @fopen($imagelink, 'r'); if ($HTTPRequest) { stream_set_timeout($HTTPRequest, 0.1); $favicon = fread($HTTPRequest, 8192); $HTTPRequestData = stream_get_meta_data($HTTPRequest); fclose($HTTPRequest); if (!$HTTPRequestData['timed_out'] && strlen($favicon) < 42) { $imagelink = ""; } } else { $imagelink = $piefeed->get_image_url(); $HTTPRequest = @fopen($imagelink, 'r'); if ($HTTPRequest) { stream_set_timeout($HTTPRequest, 0.1); $favicon = fread($HTTPRequest, 8192); $HTTPRequestData = stream_get_meta_data($HTTPRequest); fclose($HTTPRequest); if (!$HTTPRequestData['timed_out'] && strlen($favicon) < 42) { $imagelink = ""; } } else { $imagelink=""; } } $sql = "update `$FOF_FEED_TABLE` set `image`='$imagelink' where `id`='$feed_id'"; $result = fof_do_query($sql); mysql_free_result($result); } // closes image */ # # Get article items and attributes # foreach ($piefeed->get_items() as $item) { $ageflag = "0"; $dccreator = ""; $dcsubject = ""; $link = mysql_escape_string($item->get_permalink()); if (!$link) { $link = $item->get_id(); } $title = mysql_escape_string($item->get_title()); if (!$title) { $title = "[" . _("no title") . "]"; } # get <dc:creator> or <author> foreach ($item->get_authors() as $author) { $authorname = $author->get_name() . " " . $author->get_email(); if (!empty($authorname)) { $dccreator .= $authorname . ', '; } } $dccreator = mysql_escape_string(substr("{$dccreator}", 0, -2)); # get <dc:date> and <pubdate> $dcdate = mysql_escape_string($item->get_date()); $dcdate = eregi_replace(",", "", $dcdate); # get <dc:subject> or <category> $category_array = $item->get_category(); $category_array = array_unique($category_array); foreach ($category_array as $category) { if (!empty($category)) { $dcsubject .= $category . ', '; } } $dcsubject = substr(mysql_escape_string($dcsubject), 0, -2); unset($category_array); # get article content $content = mysql_escape_string($item->get_description()); $content = str_replace('"?i=http', '"http', $content); # dont know why # this creeps in if ($enclosure = $item->get_enclosure(0)) { $content .= '<br />(' . $enclosure->get_type() . '; ' . $enclosure->get_size() . ' MB)<br />'; } # # Now manage the article data # $sql = "select id from items where feed_id='{$feed_id}' and link='{$link}'"; #print "<br />" . $sql . "<br />"; $result = fof_do_query($sql); $row = mysql_fetch_array($result); $id = $row['id']; //mysql_free_result($result); # if the item does not already exist, add it if (mysql_num_rows($result) == 0) { # dcdate : August 2, 2006, 1:30 am # timestamp: 2006-09-16 15:51:53 # add it only if it's not older than keep_days $dcdatetime = strtotime($dcdate); # We set ageflag == 1 if its OK to add the item to the database if ($dcdatetime < 1) { $dcdatetime = NULL; $ageflag = 1; } else { if (time() - $dcdatetime < $keep_days * 24 * 60 * 60) { $ageflag = 1; } } if ($ageflag) { //$n++; $sql = "insert into items (feed_id,link,title,content,dcdate,dccreator,dcsubject) values ('{$feed_id}','{$link}','{$title}','{$content}','{$dcdatetime}','{$dccreator}','{$dcsubject}')"; #print "<br />" . $sql . "<br />"; $result = fof_do_query($sql); mysql_free_result($result); $ids[] = $id; #keep track of it so we don't delete it below $ageflag = 0; } } //echo "Memory usage after item foreach loop: " . number_format(memory_get_usage()); } // closes opening foreach loop //unset($piefeed); $piefeed->__destruct(); // Do what PHP should be doing on it's own. //unset($item); //unset($feed); unset($title); unset($link); unset($description); unset($content); unset($dcdatetime); unset($dccreator); unset($dcsubject); //return $n; }
function test() { $feed = new SimplePie(); $feed->set_file($this->data['file']); $feed->enable_cache(false); $feed->init(); $this->result = $feed->get_link(); }
function getFeed(&$params) { //global $mainframe; $slick_rss = array(); //init feed array if (!class_exists('SimplePie')) { //include Simple Pie processor class require_once JPATH_SITE . DS . 'libraries' . DS . 'simplepie' . DS . 'simplepie.php'; } // check if cache directory exists and is writeable $cacheDir = JPATH_BASE . DS . 'cache'; if (!is_writable($cacheDir)) { $slick_rss['error'][] = 'Cache folder is unwriteable. Solution: chmod 777 ' . $cacheDir; $cache_exists = false; } else { $cache_exists = true; } //get local module parameters from xml file module config settings $rssurl = $params->get('rssurl', NULL); $rssitems = $params->get('rssitems', 5); $rssdesc = $params->get('rssdesc', 1); $rssimage = $params->get('rssimage', 1); $rssitemtitle_words = $params->get('rssitemtitle_words', 0); $rssitemdesc = $params->get('rssitemdesc', 0); $rssitemdesc_images = $params->get('rssitemdesc_images', 1); $rssitemdesc_words = $params->get('rssitemdesc_words', 0); $rsstitle = $params->get('rsstitle', 1); $rsscache = $params->get('rsscache', 3600); $link_target = $params->get('link_target', 1); $no_follow = $params->get('no_follow', 0); $enable_tooltip = $params->get('enable_tooltip', 'yes'); $tooltip_desc_words = $params->get('t_word_count_desc', 25); $tooltip_desc_images = $params->get('tooltip_desc_images', 1); $tooltip_title_words = $params->get('t_word_count_title', 25); if (!$rssurl) { $slick_rss['error'][] = 'Invalid feed url. Please enter a valid url in the module settings.'; return $slick_rss; //halt if no valid feed url supplied } switch ($link_target) { //open links in current or new window case 1: $link_target = '_blank'; break; case 0: $link_target = '_self'; break; default: $link_target = '_blank'; break; } $slick_rss['target'] = $link_target; if ($no_follow) { $slick_rss['nofollow'] = 'rel="nofollow"'; } //Load and build the feed array $feed = new SimplePie(); $feed->set_feed_url($rssurl); //check and set caching if ($cache_exists) { $feed->set_cache_location($cacheDir); $feed->enable_cache(); $cache_time = intval($rsscache); $feed->set_cache_duration($cache_time); } else { $feed->enable_cache('false'); } $feed->init(); //process the loaded feed $feed->handle_content_type(); //store any error message if (isset($feed->error)) { $slick_rss['error'][] = $feed->error; } //start building the feed meta-info (title, desc and image) // feed title if ($feed->get_title() && $rsstitle) { $slick_rss['title']['link'] = $feed->get_link(); $slick_rss['title']['title'] = $feed->get_title(); } // feed description if ($rssdesc) { $slick_rss['description'] = $feed->get_description(); } // feed image if ($rssimage && $feed->get_image_url()) { $slick_rss['image']['url'] = $feed->get_image_url(); $slick_rss['image']['title'] = $feed->get_image_title(); } //end feed meta-info //start processing feed items //if there are items in the feed if ($feed->get_item_quantity()) { //start looping through the feed items $slick_rss_item = 0; //item counter for array indexing in the loop foreach ($feed->get_items(0, $rssitems) as $currItem) { // item title $item_title = trim($currItem->get_title()); // item title word limit check if ($rssitemtitle_words) { $item_titles = explode(' ', $item_title); $count = count($item_titles); if ($count > $rssitemtitle_words) { $item_title = ''; for ($i = 0; $i < $rssitemtitle_words; $i++) { $item_title .= ' ' . $item_titles[$i]; } $item_title .= '...'; } } $slick_rss['items'][$slick_rss_item]['title'] = $item_title; // Item Title $slick_rss['items'][$slick_rss_item]['link'] = $currItem->get_permalink(); // item description if ($rssitemdesc) { $desc = trim($currItem->get_description()); if (!$rssitemdesc_images) { $desc = preg_replace("/<img[^>]+\\>/i", "", $desc); //strip image tags } //item description word limit check if ($rssitemdesc_words) { $texts = explode(' ', $desc); $count = count($texts); if ($count > $rssitemdesc_words) { $desc = ''; for ($i = 0; $i < $rssitemdesc_words; $i++) { $desc .= ' ' . $texts[$i]; //build words } $desc .= '...'; } } $slick_rss['items'][$slick_rss_item]['description'] = $desc; //Item Description } // tooltip text if ($enable_tooltip == 'yes') { //tooltip item title $t_item_title = trim($currItem->get_title()); // tooltip title word limit check if ($tooltip_title_words) { $t_item_titles = explode(' ', $t_item_title); $count = count($t_item_titles); if ($count > $tooltip_title_words) { $tooltip_title = ''; for ($i = 0; $i < $tooltip_title_words; $i++) { $tooltip_title .= ' ' . $t_item_titles[$i]; } $tooltip_title .= '...'; } else { $tooltip_title = $t_item_title; } } else { $tooltip_title = $t_item_title; } $tooltip_title = preg_replace("/(\r\n|\n|\r)/", " ", $tooltip_title); //replace new line characters in tooltip title, important! $tooltip_title = htmlspecialchars(html_entity_decode($tooltip_title), ENT_QUOTES); //format text for tooltip $slick_rss['items'][$slick_rss_item]['tooltip']['title'] = $tooltip_title; //Tooltip Title //tooltip item description $text = trim($currItem->get_description()); if (!$tooltip_desc_images) { $text = preg_replace("/<img[^>]+\\>/i", "", $text); } // tooltip desc word limit check if ($tooltip_desc_words) { $texts = explode(' ', $text); $count = count($texts); if ($count > $tooltip_desc_words) { $text = ''; for ($i = 0; $i < $tooltip_desc_words; $i++) { $text .= ' ' . $texts[$i]; } $text .= '...'; } } $text = preg_replace("/(\r\n|\n|\r)/", " ", $text); //replace new line characters in tooltip, important! $text = htmlspecialchars(html_entity_decode($text), ENT_QUOTES); //format text for tooltip $slick_rss['items'][$slick_rss_item]['tooltip']['description'] = $text; //Tooltip Body } else { $slick_rss['items'][$slick_rss_item]['tooltip'] = array(); //blank } $slick_rss_item++; //increment item counter } } //end item quantity check if statement //return the feed data structure for the template return $slick_rss; }
<a href="?feed=http://zeldman.com/rss/" title="The father of the web standards movement">Zeldman</a></p> </div> <div id="sp_results"> <!-- As long as the feed has data to work with... --> <?php if ($success) { ?> <div class="chunk focus" align="center"> <!-- If the feed has a link back to the site that publishes it (which 99% of them do), link the feed's title to it. --> <h3 class="header"> <?php $link = $feed->get_link(); $title = $feed->get_title(); if ($link) { $title = "<a href='{$link}' title='{$title}'>{$title}</a>"; } echo $title; ?> </h3> <!-- If the feed has a description, display it. --> <?php echo $feed->get_description(); ?> </div>
function initRss($rssLink, $user) { $db = DBCxn::get(); //构建对象 $feed = new SimplePie(); $feed->set_feed_url($rssLink); //feed地址做参数进行解析操作 $feed->set_timeout(30); $feed->enable_cache(false); $feed->init(); if ($feed->error()) { //feed地址错误 //echo $feed->error(); return "error"; } $blogName = $feed->get_title(); $blogLink = $feed->get_link(); $selectOneRSS = $db->prepare("SELECT id FROM rss WHERE blogLink = ?"); $selectOneRSS->execute(array($blogLink)); $blogLinkId = $selectOneRSS->fetchColumn(); //此RSS在数据库表中id //检查是否存在,如果不存在添加此feed if ($blogLinkId > 0) { //echo "此订阅源已经存在!"; //检查用户是否已经订阅 $checkSub = $db->prepare("SELECT count(*) FROM readinfo WHERE userName = ? AND rssId = ?"); $checkSub->execute(array($user, $blogLinkId)); $boolCheckSub = $checkSub->fetchColumn(); if ($boolCheckSub != 1) { //用户没有订阅 $insertReadInfo = $db->prepare("INSERT INTO readinfo(userName,rssId) VALUES(:user,:blogLinkId)"); $insertReadInfo->bindParam(':user', $user); $insertReadInfo->bindParam(':blogLinkId', $blogLinkId); $insertReadInfo->execute(); //"此源已经存在,你之前尚未订阅,现在已经订阅"; //return "succeed1"; return "rss-" . $blogLinkId; } else { //"此源已经存在,你之前已经订阅"; return "succeed2"; } } else { $sql = "INSERT INTO rss(blogName,blogLink,rssLink,updateMd5) VALUES(:blogName,:blogLink,:rssLink,:updateMd5)"; $insertRSS = $db->prepare($sql); $insertRSS->bindParam(':blogName', $blogName); $insertRSS->bindParam(':blogLink', $blogLink); $insertRSS->bindParam(':rssLink', $rssLink); $insertRSS->bindParam(':updateMd5', $updateMd5); $updateMd5 = md5("123456"); $insertRSS->execute(); //查询该源的ID $selectOneRSS = $db->prepare("SELECT id FROM rss WHERE blogLink = ?"); $selectOneRSS->execute(array($blogLink)); $blogLinkId = $selectOneRSS->fetchColumn(); //订阅此源 //$db->exec("INSERT INTO readinfo(userName,rssId) VALUES ($user, $blogLinkId)"); $insertReadInfo = $db->prepare("INSERT INTO readinfo(userName,rssId) VALUES(:user,:blogLinkId)"); $insertReadInfo->bindParam(':user', $user); $insertReadInfo->bindParam(':blogLinkId', $blogLinkId); $insertReadInfo->execute(); //"此源不存在,现在已经订阅"; return "rss-" . $blogLinkId; } }
function parse($syncId, &$nbEvents = 0, $enableCache = true, $forceFeed = false) { $nbEvents = 0; assert('is_int($syncId) && $syncId>0'); if (empty($this->id) || 0 == $this->id) { /* Le flux ne dispose pas pas d'id !. Ça arrive si on appelle parse() sans avoir appelé save() pour un nouveau flux. @TODO: un create() pour un nouveau flux ? */ $msg = 'Empty or null id for a feed! ' . 'See ' . __FILE__ . ' on line ' . __LINE__; error_log($msg, E_USER_ERROR); die($msg); // Arrêt, sinon création événements sans flux associé. } $feed = new SimplePie(); $feed->enable_cache($enableCache); $feed->force_feed($forceFeed); $feed->set_feed_url($this->url); $feed->set_useragent('Mozilla/4.0 Leed (LightFeed Aggregator) ' . VERSION_NAME . ' by idleman http://projet.idleman.fr/leed'); if (!$feed->init()) { $this->error = $feed->error; $this->lastupdate = $_SERVER['REQUEST_TIME']; $this->save(); return false; } $feed->handle_content_type(); // UTF-8 par défaut pour SimplePie if ($this->name == '') { $this->name = $feed->get_title(); } if ($this->name == '') { $this->name = $this->url; } $this->website = $feed->get_link(); $this->description = $feed->get_description(); $items = $feed->get_items(); $eventManager = new Event(); $events = array(); $iEvents = 0; foreach ($items as $item) { // Ne retient que les 100 premiers éléments de flux. if ($iEvents++ >= 100) { break; } // Si le guid existe déjà, on évite de le reparcourir. $alreadyParsed = $eventManager->load(array('guid' => $item->get_id(), 'feed' => $this->id)); if (isset($alreadyParsed) && $alreadyParsed != false) { $events[] = $alreadyParsed->getId(); continue; } // Initialisation des informations de l'événement (élt. de flux) $event = new Event(); $event->setSyncId($syncId); $event->setGuid($item->get_id()); $event->setTitle($item->get_title()); $event->setPubdate($item->get_date()); $event->setCreator('' == $item->get_author() ? '' : $item->get_author()->name); $event->setLink($item->get_permalink()); $event->setFeed($this->id); $event->setUnread(1); // inexistant, donc non-lu //Gestion de la balise enclosure pour les podcasts et autre cochonneries :) $enclosure = $item->get_enclosure(); if ($enclosure != null && $enclosure->link != '') { $enclosureName = substr($enclosure->link, strrpos($enclosure->link, '/') + 1, strlen($enclosure->link)); $enclosureArgs = strpos($enclosureName, '?'); if ($enclosureArgs !== false) { $enclosureName = substr($enclosureName, 0, $enclosureArgs); } $enclosureFormat = isset($enclosure->handler) ? $enclosure->handler : substr($enclosureName, strrpos($enclosureName, '.') + 1); $enclosure = '<div class="enclosure"><h1>Fichier média :</h1><a href="' . $enclosure->link . '"> ' . $enclosureName . '</a> <span>(Format ' . strtoupper($enclosureFormat) . ', ' . Functions::convertFileSize($enclosure->length) . ')</span></div>'; } else { $enclosure = ''; } $event->setContent($item->get_content() . $enclosure); $event->setDescription($item->get_description() . $enclosure); if (trim($event->getDescription()) == '') { $event->setDescription(substr($event->getContent(), 0, 300) . '…<br><a href="' . $event->getLink() . '">Lire la suite de l\'article</a>'); } if (trim($event->getContent()) == '') { $event->setContent($event->getDescription()); } $event->setCategory($item->get_category()); $event->save(); $nbEvents++; } $listid = ""; foreach ($events as $item) { $listid .= ',' . $item; } $query = 'UPDATE `' . MYSQL_PREFIX . 'event` SET syncId=' . $syncId . ' WHERE id in (0' . $listid . ');'; $myQuery = $this->customQuery($query); $this->lastupdate = $_SERVER['REQUEST_TIME']; $this->save(); return true; }
/** * Fetch single feed * @param Feed feed */ public function fetch(Feed $feed) { $this->verbose('Fetching: ' . $feed->url); // Check HTTPS capability if ($feed->https == null) { $this->checkHttpsCapability($feed); } // Execute HTTP Request $request = $this->makeRequest($feed->url); if ($request['info']['http_code'] != 200) { $feed->error = '[ERROR HTTP CODE ' . $request['info']['http_code'] . ']'; $feed->fetch_interval = 60; $feed->fetched(); $feed->save(); $this->verbose('Error Fetching: ' . $feed->url); return; // skip } if (empty($request['html'])) { $feed->error = '[ERROR SERVER RETURN EMPTY CONTENT]'; $feed->fetch_interval = 60; $feed->fetched(); $feed->save(); $this->verbose('Error Fetching: ' . $feed->url); return; // skip } // Parsing feed $simplepie = new SimplePie(); // $simplepie->set_cache_location( __DIR__ . '/cache/simplepie/' ); $simplepie->set_raw_data($request['html']); $success = $simplepie->init(); if ($success === false) { $feed->error = '[ERROR PARSING FEED]'; $feed->fetch_interval = 60; $feed->fetched(); $feed->save(); $this->verbose('Error parsing: ' . $feed->url); return; // skip } $feed->title = $simplepie->get_title(); $feed->link = $simplepie->get_link(); $items = $simplepie->get_items(); $new_entries_counter = 0; foreach ($items as $item) { $entry = Entry::create(); $entry->hash = $item->get_id(true); $entry->feed_id = $feed->id; if (!$entry->exists()) { $entry->title = $item->get_title(); $entry->permalink = htmlspecialchars_decode($item->get_permalink()); $entry->content = $item->get_content(); $entry->date = $item->get_date('Y-m-d H:i:s'); if ($entry->date == null) { $entry->date = date('Y-m-d H:i:s'); } $categories = $item->get_categories(); if (!empty($categories)) { $entry_categories = array(); foreach ($categories as $category) { $entry_categories[] = $category->get_label(); } if (!empty($categories)) { $entry->categories = implode(',', $entry_categories); } } unset($categories, $entry_categories); $entry->save(); $new_entries_counter++; } } // Activity detection if ($new_entries_counter > 0) { $feed->fetch_interval = 3; } else { if ($feed->fetch_interval * 1.5 <= 20) { $feed->fetch_interval = round($feed->fetch_interval * 1.5); } } $feed->error = null; $feed->fetched(); $feed->save(); $this->getFavicon($feed); }
<a href="?feed=http://youtube.com/rss/global/top_favorites.rss" title="Funny user-submitted videos">You Tube</a>, <a href="?feed=http://zeldman.com/rss/" title="The father of the web standards movement">Zeldman</a></p> </div> <div id="sp_results"> <!-- As long as the feed has data to work with... --> <?php if ($success) { ?> <div class="chunk focus" align="center"> <!-- If the feed has a link back to the site that publishes it (which 99% of them do), link the feed's title to it. --> <h3 class="header"><?php if ($feed->get_link()) { echo '<a href="' . $feed->get_link() . '">'; } echo $feed->get_title(); if ($feed->get_link()) { echo '</a>'; } ?> </h3> <!-- If the feed has a description, display it. --> <?php echo $feed->get_description(); ?> </div>
/** * Add a new feed to the database * * Adds the specified feed name and URL to the global <tt>$data</tt> array. If no name is set * by the user, it fetches one from the feed. If the URL specified is a HTML page and not a * feed, it lets SimplePie do autodiscovery and uses the XML url returned. * * @since 1.0 * @uses $data Contains all feeds, this is what we add the new feed to * * @param string $url URL to feed or website (if autodiscovering) * @param string $name Title/Name of feed * @param string $cat Category to add feed to * @param bool $return If true, return the new feed's details. Otherwise, use the global $data array * @return bool True if succeeded, false if failed */ function add_feed($url, $name = '', $cat = 'default', $return = false) { if (empty($url)) { throw new Exception(_r("Couldn't add feed: No feed URL supplied"), Errors::get_code('admin.feeds.no_url')); } require_once LILINA_INCPATH . '/contrib/simplepie/simplepie.inc'; $feed_info = new SimplePie(); $feed_info->set_useragent('Lilina/' . LILINA_CORE_VERSION . '; (' . get_option('baseurl') . '; http://getlilina.org/; Allow Like Gecko) SimplePie/' . SIMPLEPIE_BUILD); $feed_info->set_stupidly_fast(true); $feed_info->enable_cache(false); $feed_info->set_feed_url(urldecode($url)); $feed_info->init(); $feed_error = $feed_info->error(); $feed_url = $feed_info->subscribe_url(); if (!empty($feed_error)) { //No feeds autodiscovered; throw new Exception(sprintf(_r("Couldn't add feed: %s is not a valid URL or the server could not be accessed. Additionally, no feeds could be found by autodiscovery."), $url), Errors::get_code('admin.feeds.invalid_url')); } if (empty($name)) { //Get it from the feed $name = $feed_info->get_title(); } if ($return === true) { return array('feed' => $feed_url, 'url' => $feed_info->get_link(), 'name' => $name, 'cat' => $cat); } global $data; $data['feeds'][] = array('feed' => $feed_url, 'url' => $feed_info->get_link(), 'name' => $name, 'cat' => $cat); save_feeds(); return sprintf(_r('Added feed "%1$s"'), $name); }
public function getImageUrl() { $url = parse_url($this->simplePie->get_link()); $url = urlencode(sprintf('%s://%s', $url['scheme'], $url['host'])); return "http://g.etfv.co/{$url}"; }
function update_rss_feed($link, $feed, $ignore_daemon = false, $no_cache = false, $override_url = false) { require_once "lib/simplepie/simplepie.inc"; require_once "lib/magpierss/rss_fetch.inc"; require_once 'lib/magpierss/rss_utils.inc'; $debug_enabled = defined('DAEMON_EXTENDED_DEBUG') || $_REQUEST['xdebug']; if (!$_REQUEST["daemon"] && !$ignore_daemon) { return false; } if ($debug_enabled) { _debug("update_rss_feed: start"); } if (!$ignore_daemon) { if (DB_TYPE == "pgsql") { $updstart_thresh_qpart = "(ttrss_feeds.last_update_started IS NULL OR ttrss_feeds.last_update_started < NOW() - INTERVAL '120 seconds')"; } else { $updstart_thresh_qpart = "(ttrss_feeds.last_update_started IS NULL OR ttrss_feeds.last_update_started < DATE_SUB(NOW(), INTERVAL 120 SECOND))"; } $result = db_query($link, "SELECT id,update_interval,auth_login,\n\t\t\t\tauth_pass,cache_images,update_method,last_updated\n\t\t\t\tFROM ttrss_feeds WHERE id = '{$feed}' AND {$updstart_thresh_qpart}"); } else { $result = db_query($link, "SELECT id,update_interval,auth_login,\n\t\t\t\tfeed_url,auth_pass,cache_images,update_method,last_updated,\n\t\t\t\tmark_unread_on_update, owner_uid, update_on_checksum_change,\n\t\t\t\tpubsub_state\n\t\t\t\tFROM ttrss_feeds WHERE id = '{$feed}'"); } if (db_num_rows($result) == 0) { if ($debug_enabled) { _debug("update_rss_feed: feed {$feed} NOT FOUND/SKIPPED"); } return false; } $update_method = db_fetch_result($result, 0, "update_method"); $last_updated = db_fetch_result($result, 0, "last_updated"); $owner_uid = db_fetch_result($result, 0, "owner_uid"); $mark_unread_on_update = sql_bool_to_bool(db_fetch_result($result, 0, "mark_unread_on_update")); $update_on_checksum_change = sql_bool_to_bool(db_fetch_result($result, 0, "update_on_checksum_change")); $pubsub_state = db_fetch_result($result, 0, "pubsub_state"); db_query($link, "UPDATE ttrss_feeds SET last_update_started = NOW()\n\t\t\tWHERE id = '{$feed}'"); $auth_login = db_fetch_result($result, 0, "auth_login"); $auth_pass = db_fetch_result($result, 0, "auth_pass"); if ($update_method == 0) { $update_method = DEFAULT_UPDATE_METHOD + 1; } // 1 - Magpie // 2 - SimplePie // 3 - Twitter OAuth if ($update_method == 2) { $use_simplepie = true; } else { $use_simplepie = false; } if ($debug_enabled) { _debug("update method: {$update_method} (feed setting: {$update_method}) (use simplepie: {$use_simplepie})\n"); } if ($update_method == 1) { $auth_login = urlencode($auth_login); $auth_pass = urlencode($auth_pass); } $cache_images = sql_bool_to_bool(db_fetch_result($result, 0, "cache_images")); $fetch_url = db_fetch_result($result, 0, "feed_url"); $feed = db_escape_string($feed); if ($auth_login && $auth_pass) { $url_parts = array(); preg_match("/(^[^:]*):\\/\\/(.*)/", $fetch_url, $url_parts); if ($url_parts[1] && $url_parts[2]) { $fetch_url = $url_parts[1] . "://{$auth_login}:{$auth_pass}@" . $url_parts[2]; } } if ($override_url) { $fetch_url = $override_url; } if ($debug_enabled) { _debug("update_rss_feed: fetching [{$fetch_url}]..."); } // Ignore cache if new feed or manual update. $cache_age = is_null($last_updated) || $last_updated == '1970-01-01 00:00:00' ? -1 : get_feed_update_interval($link, $feed) * 60; if ($update_method == 3) { $rss = fetch_twitter_rss($link, $fetch_url, $owner_uid); } else { if ($update_method == 1) { define('MAGPIE_CACHE_AGE', $cache_age); define('MAGPIE_CACHE_ON', !$no_cache); define('MAGPIE_FETCH_TIME_OUT', 60); define('MAGPIE_CACHE_DIR', CACHE_DIR . "/magpie"); $rss = @fetch_rss($fetch_url); } else { $simplepie_cache_dir = CACHE_DIR . "/simplepie"; if (!is_dir($simplepie_cache_dir)) { mkdir($simplepie_cache_dir); } $rss = new SimplePie(); $rss->set_useragent(SELF_USER_AGENT); # $rss->set_timeout(10); $rss->set_feed_url($fetch_url); $rss->set_output_encoding('UTF-8'); //$rss->force_feed(true); if ($debug_enabled) { _debug("feed update interval (sec): " . get_feed_update_interval($link, $feed) * 60); } $rss->enable_cache(!$no_cache); if (!$no_cache) { $rss->set_cache_location($simplepie_cache_dir); $rss->set_cache_duration($cache_age); } $rss->init(); } } // print_r($rss); if ($debug_enabled) { _debug("update_rss_feed: fetch done, parsing..."); } $feed = db_escape_string($feed); if ($update_method == 2) { $fetch_ok = !$rss->error(); } else { $fetch_ok = !!$rss; } if ($fetch_ok) { if ($debug_enabled) { _debug("update_rss_feed: processing feed data..."); } // db_query($link, "BEGIN"); if (DB_TYPE == "pgsql") { $favicon_interval_qpart = "favicon_last_checked < NOW() - INTERVAL '12 hour'"; } else { $favicon_interval_qpart = "favicon_last_checked < DATE_SUB(NOW(), INTERVAL 12 HOUR)"; } $result = db_query($link, "SELECT title,icon_url,site_url,owner_uid,\n\t\t\t\t(favicon_last_checked IS NULL OR {$favicon_interval_qpart}) AS\n\t\t\t\t\t\tfavicon_needs_check\n\t\t\t\tFROM ttrss_feeds WHERE id = '{$feed}'"); $registered_title = db_fetch_result($result, 0, "title"); $orig_icon_url = db_fetch_result($result, 0, "icon_url"); $orig_site_url = db_fetch_result($result, 0, "site_url"); $favicon_needs_check = sql_bool_to_bool(db_fetch_result($result, 0, "favicon_needs_check")); $owner_uid = db_fetch_result($result, 0, "owner_uid"); if ($use_simplepie) { $site_url = db_escape_string(trim($rss->get_link())); } else { $site_url = db_escape_string(trim($rss->channel["link"])); } // weird, weird Magpie if (!$use_simplepie) { if (!$site_url) { $site_url = db_escape_string($rss->channel["link_"]); } } $site_url = rewrite_relative_url($fetch_url, $site_url); $site_url = substr($site_url, 0, 250); if ($debug_enabled) { _debug("update_rss_feed: checking favicon..."); } if ($favicon_needs_check) { check_feed_favicon($site_url, $feed, $link); db_query($link, "UPDATE ttrss_feeds SET favicon_last_checked = NOW()\n\t\t\t\t\tWHERE id = '{$feed}'"); } if (!$registered_title || $registered_title == "[Unknown]") { if ($use_simplepie) { $feed_title = db_escape_string($rss->get_title()); } else { $feed_title = db_escape_string($rss->channel["title"]); } if ($debug_enabled) { _debug("update_rss_feed: registering title: {$feed_title}"); } db_query($link, "UPDATE ttrss_feeds SET\n\t\t\t\t\ttitle = '{$feed_title}' WHERE id = '{$feed}'"); } if ($site_url && $orig_site_url != $site_url) { db_query($link, "UPDATE ttrss_feeds SET\n\t\t\t\t\tsite_url = '{$site_url}' WHERE id = '{$feed}'"); } // print "I: " . $rss->channel["image"]["url"]; if (!$use_simplepie) { $icon_url = db_escape_string(trim($rss->image["url"])); } else { $icon_url = db_escape_string(trim($rss->get_image_url())); } $icon_url = rewrite_relative_url($fetch_url, $icon_url); $icon_url = substr($icon_url, 0, 250); if ($icon_url && $orig_icon_url != $icon_url) { db_query($link, "UPDATE ttrss_feeds SET icon_url = '{$icon_url}' WHERE id = '{$feed}'"); } if ($debug_enabled) { _debug("update_rss_feed: loading filters..."); } $filters = load_filters($link, $feed, $owner_uid); // if ($debug_enabled) { // print_r($filters); // } if ($use_simplepie) { $iterator = $rss->get_items(); } else { $iterator = $rss->items; if (!$iterator || !is_array($iterator)) { $iterator = $rss->entries; } if (!$iterator || !is_array($iterator)) { $iterator = $rss; } } if (!is_array($iterator)) { /* db_query($link, "UPDATE ttrss_feeds SET last_error = 'Parse error: can\'t find any articles.' WHERE id = '$feed'"); */ // clear any errors and mark feed as updated if fetched okay // even if it's blank if ($debug_enabled) { _debug("update_rss_feed: entry iterator is not an array, no articles?"); } db_query($link, "UPDATE ttrss_feeds\n\t\t\t\t\tSET last_updated = NOW(), last_error = '' WHERE id = '{$feed}'"); return; // no articles } if ($pubsub_state != 2 && PUBSUBHUBBUB_ENABLED) { if ($debug_enabled) { _debug("update_rss_feed: checking for PUSH hub..."); } $feed_hub_url = false; if ($use_simplepie) { $links = $rss->get_links('hub'); if ($links && is_array($links)) { foreach ($links as $l) { $feed_hub_url = $l; break; } } } else { $atom = $rss->channel['atom']; if ($atom) { if ($atom['link@rel'] == 'hub') { $feed_hub_url = $atom['link@href']; } if (!$feed_hub_url && $atom['link#'] > 1) { for ($i = 2; $i <= $atom['link#']; $i++) { if ($atom["link#{$i}@rel"] == 'hub') { $feed_hub_url = $atom["link#{$i}@href"]; break; } } } } else { $feed_hub_url = $rss->channel['link_hub']; } } if ($debug_enabled) { _debug("update_rss_feed: feed hub url: {$feed_hub_url}"); } if ($feed_hub_url && function_exists('curl_init') && !ini_get("open_basedir")) { require_once 'lib/pubsubhubbub/subscriber.php'; $callback_url = get_self_url_prefix() . "/public.php?op=pubsub&id={$feed}"; $s = new Subscriber($feed_hub_url, $callback_url); $rc = $s->subscribe($fetch_url); if ($debug_enabled) { _debug("update_rss_feed: feed hub url found, subscribe request sent."); } db_query($link, "UPDATE ttrss_feeds SET pubsub_state = 1\n\t\t\t\t\t\tWHERE id = '{$feed}'"); } } if ($debug_enabled) { _debug("update_rss_feed: processing articles..."); } foreach ($iterator as $item) { if ($_REQUEST['xdebug'] == 2) { print_r($item); } if ($use_simplepie) { $entry_guid = $item->get_id(); if (!$entry_guid) { $entry_guid = $item->get_link(); } if (!$entry_guid) { $entry_guid = make_guid_from_title($item->get_title()); } } else { $entry_guid = $item["id"]; if (!$entry_guid) { $entry_guid = $item["guid"]; } if (!$entry_guid) { $entry_guid = $item["about"]; } if (!$entry_guid) { $entry_guid = $item["link"]; } if (!$entry_guid) { $entry_guid = make_guid_from_title($item["title"]); } } if ($debug_enabled) { _debug("update_rss_feed: guid {$entry_guid}"); } if (!$entry_guid) { continue; } $entry_timestamp = ""; if ($use_simplepie) { $entry_timestamp = strtotime($item->get_date()); } else { $rss_2_date = $item['pubdate']; $rss_1_date = $item['dc']['date']; $atom_date = $item['issued']; if (!$atom_date) { $atom_date = $item['updated']; } if ($atom_date != "") { $entry_timestamp = parse_w3cdtf($atom_date); } if ($rss_1_date != "") { $entry_timestamp = parse_w3cdtf($rss_1_date); } if ($rss_2_date != "") { $entry_timestamp = strtotime($rss_2_date); } } if ($entry_timestamp == "" || $entry_timestamp == -1 || !$entry_timestamp) { $entry_timestamp = time(); $no_orig_date = 'true'; } else { $no_orig_date = 'false'; } $entry_timestamp_fmt = strftime("%Y/%m/%d %H:%M:%S", $entry_timestamp); if ($debug_enabled) { _debug("update_rss_feed: date {$entry_timestamp} [{$entry_timestamp_fmt}]"); } if ($use_simplepie) { $entry_title = $item->get_title(); } else { $entry_title = trim(strip_tags($item["title"])); } if ($use_simplepie) { $entry_link = $item->get_link(); } else { // strange Magpie workaround $entry_link = $item["link_"]; if (!$entry_link) { $entry_link = $item["link"]; } } $entry_link = rewrite_relative_url($site_url, $entry_link); if ($debug_enabled) { _debug("update_rss_feed: title {$entry_title}"); _debug("update_rss_feed: link {$entry_link}"); } if (!$entry_title) { $entry_title = date("Y-m-d H:i:s", $entry_timestamp); } $entry_link = strip_tags($entry_link); if ($use_simplepie) { $entry_content = $item->get_content(); if (!$entry_content) { $entry_content = $item->get_description(); } } else { $entry_content = $item["content:escaped"]; if (!$entry_content) { $entry_content = $item["content:encoded"]; } if (!$entry_content && is_array($entry_content)) { $entry_content = $item["content"]["encoded"]; } if (!$entry_content) { $entry_content = $item["content"]; } if (is_array($entry_content)) { $entry_content = $entry_content[0]; } // Magpie bugs are getting ridiculous if (trim($entry_content) == "Array") { $entry_content = false; } if (!$entry_content) { $entry_content = $item["atom_content"]; } if (!$entry_content) { $entry_content = $item["summary"]; } if (!$entry_content || strlen($entry_content) < strlen($item["description"])) { $entry_content = $item["description"]; } // WTF if (is_array($entry_content)) { $entry_content = $entry_content["encoded"]; if (!$entry_content) { $entry_content = $entry_content["escaped"]; } } } if ($cache_images && is_writable(CACHE_DIR . '/images')) { $entry_content = cache_images($entry_content, $site_url, $debug_enabled); } if ($_REQUEST["xdebug"] == 2) { print "update_rss_feed: content: "; print $entry_content; print "\n"; } $entry_content_unescaped = $entry_content; if ($use_simplepie) { $entry_comments = strip_tags($item->data["comments"]); if ($item->get_author()) { $entry_author_item = $item->get_author(); $entry_author = $entry_author_item->get_name(); if (!$entry_author) { $entry_author = $entry_author_item->get_email(); } $entry_author = db_escape_string($entry_author); } } else { $entry_comments = strip_tags($item["comments"]); $entry_author = db_escape_string(strip_tags($item['dc']['creator'])); if ($item['author']) { if (is_array($item['author'])) { if (!$entry_author) { $entry_author = db_escape_string(strip_tags($item['author']['name'])); } if (!$entry_author) { $entry_author = db_escape_string(strip_tags($item['author']['email'])); } } if (!$entry_author) { $entry_author = db_escape_string(strip_tags($item['author'])); } } } if (preg_match('/^[\\t\\n\\r ]*$/', $entry_author)) { $entry_author = ''; } $entry_guid = db_escape_string(strip_tags($entry_guid)); $entry_guid = mb_substr($entry_guid, 0, 250); $result = db_query($link, "SELECT id FROM\tttrss_entries\n\t\t\t\t\tWHERE guid = '{$entry_guid}'"); $entry_content = db_escape_string($entry_content, false); $content_hash = "SHA1:" . sha1(strip_tags($entry_content)); $entry_title = db_escape_string($entry_title); $entry_link = db_escape_string($entry_link); $entry_comments = mb_substr(db_escape_string($entry_comments), 0, 250); $entry_author = mb_substr($entry_author, 0, 250); if ($use_simplepie) { $num_comments = 0; #FIXME# } else { $num_comments = db_escape_string($item["slash"]["comments"]); } if (!$num_comments) { $num_comments = 0; } if ($debug_enabled) { _debug("update_rss_feed: looking for tags [1]..."); } // parse <category> entries into tags $additional_tags = array(); if ($use_simplepie) { $additional_tags_src = $item->get_categories(); if (is_array($additional_tags_src)) { foreach ($additional_tags_src as $tobj) { array_push($additional_tags, $tobj->get_term()); } } if ($debug_enabled) { _debug("update_rss_feed: category tags:"); print_r($additional_tags); } } else { $t_ctr = $item['category#']; if ($t_ctr == 0) { $additional_tags = array(); } else { if ($t_ctr > 0) { $additional_tags = array($item['category']); if ($item['category@term']) { array_push($additional_tags, $item['category@term']); } for ($i = 0; $i <= $t_ctr; $i++) { if ($item["category#{$i}"]) { array_push($additional_tags, $item["category#{$i}"]); } if ($item["category#{$i}@term"]) { array_push($additional_tags, $item["category#{$i}@term"]); } } } } // parse <dc:subject> elements $t_ctr = $item['dc']['subject#']; if ($t_ctr > 0) { array_push($additional_tags, $item['dc']['subject']); for ($i = 0; $i <= $t_ctr; $i++) { if ($item['dc']["subject#{$i}"]) { array_push($additional_tags, $item['dc']["subject#{$i}"]); } } } } if ($debug_enabled) { _debug("update_rss_feed: looking for tags [2]..."); } /* taaaags */ // <a href="..." rel="tag">Xorg</a>, // $entry_tags = null; preg_match_all("/<a.*?rel=['\"]tag['\"].*?\\>([^<]+)<\\/a>/i", $entry_content_unescaped, $entry_tags); $entry_tags = $entry_tags[1]; $entry_tags = array_merge($entry_tags, $additional_tags); $entry_tags = array_unique($entry_tags); for ($i = 0; $i < count($entry_tags); $i++) { $entry_tags[$i] = mb_strtolower($entry_tags[$i], 'utf-8'); } if ($debug_enabled) { //_debug("update_rss_feed: unfiltered tags found:"); //print_r($entry_tags); } # sanitize content $entry_content = sanitize_article_content($entry_content); $entry_title = sanitize_article_content($entry_title); if ($debug_enabled) { _debug("update_rss_feed: done collecting data [TITLE:{$entry_title}]"); } db_query($link, "BEGIN"); if (db_num_rows($result) == 0) { if ($debug_enabled) { _debug("update_rss_feed: base guid not found"); } // base post entry does not exist, create it $result = db_query($link, "INSERT INTO ttrss_entries\n\t\t\t\t\t\t\t(title,\n\t\t\t\t\t\t\tguid,\n\t\t\t\t\t\t\tlink,\n\t\t\t\t\t\t\tupdated,\n\t\t\t\t\t\t\tcontent,\n\t\t\t\t\t\t\tcontent_hash,\n\t\t\t\t\t\t\tno_orig_date,\n\t\t\t\t\t\t\tdate_updated,\n\t\t\t\t\t\t\tdate_entered,\n\t\t\t\t\t\t\tcomments,\n\t\t\t\t\t\t\tnum_comments,\n\t\t\t\t\t\t\tauthor)\n\t\t\t\t\t\tVALUES\n\t\t\t\t\t\t\t('{$entry_title}',\n\t\t\t\t\t\t\t'{$entry_guid}',\n\t\t\t\t\t\t\t'{$entry_link}',\n\t\t\t\t\t\t\t'{$entry_timestamp_fmt}',\n\t\t\t\t\t\t\t'{$entry_content}',\n\t\t\t\t\t\t\t'{$content_hash}',\n\t\t\t\t\t\t\t{$no_orig_date},\n\t\t\t\t\t\t\tNOW(),\n\t\t\t\t\t\t\tNOW(),\n\t\t\t\t\t\t\t'{$entry_comments}',\n\t\t\t\t\t\t\t'{$num_comments}',\n\t\t\t\t\t\t\t'{$entry_author}')"); } else { // we keep encountering the entry in feeds, so we need to // update date_updated column so that we don't get horrible // dupes when the entry gets purged and reinserted again e.g. // in the case of SLOW SLOW OMG SLOW updating feeds $base_entry_id = db_fetch_result($result, 0, "id"); db_query($link, "UPDATE ttrss_entries SET date_updated = NOW()\n\t\t\t\t\t\tWHERE id = '{$base_entry_id}'"); } // now it should exist, if not - bad luck then $result = db_query($link, "SELECT\n\t\t\t\t\t\tid,content_hash,no_orig_date,title,\n\t\t\t\t\t\t" . SUBSTRING_FOR_DATE . "(date_updated,1,19) as date_updated,\n\t\t\t\t\t\t" . SUBSTRING_FOR_DATE . "(updated,1,19) as updated,\n\t\t\t\t\t\tnum_comments\n\t\t\t\t\tFROM\n\t\t\t\t\t\tttrss_entries\n\t\t\t\t\tWHERE guid = '{$entry_guid}'"); $entry_ref_id = 0; $entry_int_id = 0; if (db_num_rows($result) == 1) { if ($debug_enabled) { _debug("update_rss_feed: base guid found, checking for user record"); } // this will be used below in update handler $orig_content_hash = db_fetch_result($result, 0, "content_hash"); $orig_title = db_fetch_result($result, 0, "title"); $orig_num_comments = db_fetch_result($result, 0, "num_comments"); $orig_date_updated = strtotime(db_fetch_result($result, 0, "date_updated")); $ref_id = db_fetch_result($result, 0, "id"); $entry_ref_id = $ref_id; // check for user post link to main table // do we allow duplicate posts with same GUID in different feeds? if (get_pref($link, "ALLOW_DUPLICATE_POSTS", $owner_uid, false)) { $dupcheck_qpart = "AND (feed_id = '{$feed}' OR feed_id IS NULL)"; } else { $dupcheck_qpart = ""; } /* Collect article tags here so we could filter by them: */ $article_filters = get_article_filters($filters, $entry_title, $entry_content, $entry_link, $entry_timestamp, $entry_author, $entry_tags); if ($debug_enabled) { _debug("update_rss_feed: article filters: "); if (count($article_filters) != 0) { print_r($article_filters); } } if (find_article_filter($article_filters, "filter")) { db_query($link, "COMMIT"); // close transaction in progress continue; } $score = calculate_article_score($article_filters); if ($debug_enabled) { _debug("update_rss_feed: initial score: {$score}"); } $query = "SELECT ref_id, int_id FROM ttrss_user_entries WHERE\n\t\t\t\t\t\t\tref_id = '{$ref_id}' AND owner_uid = '{$owner_uid}'\n\t\t\t\t\t\t\t{$dupcheck_qpart}"; // if ($_REQUEST["xdebug"]) print "$query\n"; $result = db_query($link, $query); // okay it doesn't exist - create user entry if (db_num_rows($result) == 0) { if ($debug_enabled) { _debug("update_rss_feed: user record not found, creating..."); } if ($score >= -500 && !find_article_filter($article_filters, 'catchup')) { $unread = 'true'; $last_read_qpart = 'NULL'; } else { $unread = 'false'; $last_read_qpart = 'NOW()'; } if (find_article_filter($article_filters, 'mark') || $score > 1000) { $marked = 'true'; } else { $marked = 'false'; } if (find_article_filter($article_filters, 'publish')) { $published = 'true'; } else { $published = 'false'; } // N-grams if (DB_TYPE == "pgsql" and defined('_NGRAM_TITLE_DUPLICATE_THRESHOLD')) { $result = db_query($link, "SELECT COUNT(*) AS similar FROM\n\t\t\t\t\t\t\t\t\tttrss_entries,ttrss_user_entries\n\t\t\t\t\t\t\t\tWHERE ref_id = id AND updated >= NOW() - INTERVAL '7 day'\n\t\t\t\t\t\t\t\t\tAND similarity(title, '{$entry_title}') >= " . _NGRAM_TITLE_DUPLICATE_THRESHOLD . "\n\t\t\t\t\t\t\t\t\tAND owner_uid = {$owner_uid}"); $ngram_similar = db_fetch_result($result, 0, "similar"); if ($debug_enabled) { _debug("update_rss_feed: N-gram similar results: {$ngram_similar}"); } if ($ngram_similar > 0) { $unread = 'false'; } } $result = db_query($link, "INSERT INTO ttrss_user_entries\n\t\t\t\t\t\t\t\t(ref_id, owner_uid, feed_id, unread, last_read, marked,\n\t\t\t\t\t\t\t\t\tpublished, score, tag_cache, label_cache, uuid)\n\t\t\t\t\t\t\tVALUES ('{$ref_id}', '{$owner_uid}', '{$feed}', {$unread},\n\t\t\t\t\t\t\t\t{$last_read_qpart}, {$marked}, {$published}, '{$score}', '', '', '')"); if (PUBSUBHUBBUB_HUB && $published == 'true') { $rss_link = get_self_url_prefix() . "/public.php?op=rss&id=-2&key=" . get_feed_access_key($link, -2, false, $owner_uid); $p = new Publisher(PUBSUBHUBBUB_HUB); $pubsub_result = $p->publish_update($rss_link); } $result = db_query($link, "SELECT int_id FROM ttrss_user_entries WHERE\n\t\t\t\t\t\t\t\tref_id = '{$ref_id}' AND owner_uid = '{$owner_uid}' AND\n\t\t\t\t\t\t\t\tfeed_id = '{$feed}' LIMIT 1"); if (db_num_rows($result) == 1) { $entry_int_id = db_fetch_result($result, 0, "int_id"); } } else { if ($debug_enabled) { _debug("update_rss_feed: user record FOUND"); } $entry_ref_id = db_fetch_result($result, 0, "ref_id"); $entry_int_id = db_fetch_result($result, 0, "int_id"); } if ($debug_enabled) { _debug("update_rss_feed: RID: {$entry_ref_id}, IID: {$entry_int_id}"); } $post_needs_update = false; $update_insignificant = false; if ($orig_num_comments != $num_comments) { $post_needs_update = true; $update_insignificant = true; } if ($content_hash != $orig_content_hash) { $post_needs_update = true; $update_insignificant = false; } if (db_escape_string($orig_title) != $entry_title) { $post_needs_update = true; $update_insignificant = false; } // if post needs update, update it and mark all user entries // linking to this post as updated if ($post_needs_update) { if (defined('DAEMON_EXTENDED_DEBUG')) { _debug("update_rss_feed: post {$entry_guid} needs update..."); } // print "<!-- post $orig_title needs update : $post_needs_update -->"; db_query($link, "UPDATE ttrss_entries\n\t\t\t\t\t\t\tSET title = '{$entry_title}', content = '{$entry_content}',\n\t\t\t\t\t\t\t\tcontent_hash = '{$content_hash}',\n\t\t\t\t\t\t\t\tupdated = '{$entry_timestamp_fmt}',\n\t\t\t\t\t\t\t\tnum_comments = '{$num_comments}'\n\t\t\t\t\t\t\tWHERE id = '{$ref_id}'"); if (!$update_insignificant) { if ($mark_unread_on_update) { db_query($link, "UPDATE ttrss_user_entries\n\t\t\t\t\t\t\t\t\tSET last_read = null, unread = true WHERE ref_id = '{$ref_id}'"); } else { if ($update_on_checksum_change) { db_query($link, "UPDATE ttrss_user_entries\n\t\t\t\t\t\t\t\t\tSET last_read = null WHERE ref_id = '{$ref_id}'\n\t\t\t\t\t\t\t\t\t\tAND unread = false"); } } } } } db_query($link, "COMMIT"); if ($debug_enabled) { _debug("update_rss_feed: assigning labels..."); } assign_article_to_labels($link, $entry_ref_id, $article_filters, $owner_uid); if ($debug_enabled) { _debug("update_rss_feed: looking for enclosures..."); } // enclosures $enclosures = array(); if ($use_simplepie) { $encs = $item->get_enclosures(); if (is_array($encs)) { foreach ($encs as $e) { $e_item = array($e->link, $e->type, $e->length); array_push($enclosures, $e_item); } } } else { // <enclosure> $e_ctr = $item['enclosure#']; if ($e_ctr > 0) { $e_item = array($item['enclosure@url'], $item['enclosure@type'], $item['enclosure@length']); array_push($enclosures, $e_item); for ($i = 0; $i <= $e_ctr; $i++) { if ($item["enclosure#{$i}@url"]) { $e_item = array($item["enclosure#{$i}@url"], $item["enclosure#{$i}@type"], $item["enclosure#{$i}@length"]); array_push($enclosures, $e_item); } } } // <media:content> // can there be many of those? yes -fox $m_ctr = $item['media']['content#']; if ($m_ctr > 0) { $e_item = array($item['media']['content@url'], $item['media']['content@medium'], $item['media']['content@length']); array_push($enclosures, $e_item); for ($i = 0; $i <= $m_ctr; $i++) { if ($item["media"]["content#{$i}@url"]) { $e_item = array($item["media"]["content#{$i}@url"], $item["media"]["content#{$i}@medium"], $item["media"]["content#{$i}@length"]); array_push($enclosures, $e_item); } } } } if ($debug_enabled) { _debug("update_rss_feed: article enclosures:"); print_r($enclosures); } db_query($link, "BEGIN"); foreach ($enclosures as $enc) { $enc_url = db_escape_string($enc[0]); $enc_type = db_escape_string($enc[1]); $enc_dur = db_escape_string($enc[2]); $result = db_query($link, "SELECT id FROM ttrss_enclosures\n\t\t\t\t\t\tWHERE content_url = '{$enc_url}' AND post_id = '{$entry_ref_id}'"); if (db_num_rows($result) == 0) { db_query($link, "INSERT INTO ttrss_enclosures\n\t\t\t\t\t\t\t(content_url, content_type, title, duration, post_id) VALUES\n\t\t\t\t\t\t\t('{$enc_url}', '{$enc_type}', '', '{$enc_dur}', '{$entry_ref_id}')"); } } db_query($link, "COMMIT"); // check for manual tags (we have to do it here since they're loaded from filters) foreach ($article_filters as $f) { if ($f["type"] == "tag") { $manual_tags = trim_array(explode(",", $f["param"])); foreach ($manual_tags as $tag) { if (tag_is_valid($tag)) { array_push($entry_tags, $tag); } } } } // Skip boring tags $boring_tags = trim_array(explode(",", mb_strtolower(get_pref($link, 'BLACKLISTED_TAGS', $owner_uid, ''), 'utf-8'))); $filtered_tags = array(); $tags_to_cache = array(); if ($entry_tags && is_array($entry_tags)) { foreach ($entry_tags as $tag) { if (array_search($tag, $boring_tags) === false) { array_push($filtered_tags, $tag); } } } $filtered_tags = array_unique($filtered_tags); if ($debug_enabled) { _debug("update_rss_feed: filtered article tags:"); print_r($filtered_tags); } // Save article tags in the database if (count($filtered_tags) > 0) { db_query($link, "BEGIN"); foreach ($filtered_tags as $tag) { $tag = sanitize_tag($tag); $tag = db_escape_string($tag); if (!tag_is_valid($tag)) { continue; } $result = db_query($link, "SELECT id FROM ttrss_tags\n\t\t\t\t\t\t\tWHERE tag_name = '{$tag}' AND post_int_id = '{$entry_int_id}' AND\n\t\t\t\t\t\t\towner_uid = '{$owner_uid}' LIMIT 1"); if ($result && db_num_rows($result) == 0) { db_query($link, "INSERT INTO ttrss_tags\n\t\t\t\t\t\t\t\t\t(owner_uid,tag_name,post_int_id)\n\t\t\t\t\t\t\t\t\tVALUES ('{$owner_uid}','{$tag}', '{$entry_int_id}')"); } array_push($tags_to_cache, $tag); } /* update the cache */ $tags_to_cache = array_unique($tags_to_cache); $tags_str = db_escape_string(join(",", $tags_to_cache)); db_query($link, "UPDATE ttrss_user_entries\n\t\t\t\t\t\tSET tag_cache = '{$tags_str}' WHERE ref_id = '{$entry_ref_id}'\n\t\t\t\t\t\tAND owner_uid = {$owner_uid}"); db_query($link, "COMMIT"); } if ($debug_enabled) { _debug("update_rss_feed: article processed"); } } if (!$last_updated) { if ($debug_enabled) { _debug("update_rss_feed: new feed, catching it up..."); } catchup_feed($link, $feed, false, $owner_uid); } if ($debug_enabled) { _debug("purging feed..."); } purge_feed($link, $feed, 0, $debug_enabled); db_query($link, "UPDATE ttrss_feeds\n\t\t\t\tSET last_updated = NOW(), last_error = '' WHERE id = '{$feed}'"); // db_query($link, "COMMIT"); } else { if ($use_simplepie) { $error_msg = mb_substr($rss->error(), 0, 250); } else { $error_msg = mb_substr(magpie_error(), 0, 250); } if ($debug_enabled) { _debug("update_rss_feed: error fetching feed: {$error_msg}"); } $error_msg = db_escape_string($error_msg); db_query($link, "UPDATE ttrss_feeds SET last_error = '{$error_msg}',\n\t\t\t\t\tlast_updated = NOW() WHERE id = '{$feed}'"); } if ($use_simplepie) { unset($rss); } if ($debug_enabled) { _debug("update_rss_feed: done"); } }
/** * getFeed * * @param array &$params The module options. * * @return void */ public static function getFeed(&$params) { // Global $mainframe; // Init feed array $light_rss = array(); // Get local module parameters from xml file module config settings $rssurl = $params->get('rssurl', null); $rssitems = $params->get('rssitems', 5); $rssdesc = $params->get('rssdesc', 1); $rssimage = $params->get('rssimage', 1); $rssitemtitle_words = $params->get('rssitemtitle_words', 0); $rssitemdesc = $params->get('rssitemdesc', 0); $rssitemdesc_images = $params->get('rssitemdesc_images', 1); $rssitemdesc_words = $params->get('rssitemdesc_words', 0); $rsstitle = $params->get('rsstitle', 1); $rsscache = $params->get('rsscache', 3600); $link_target = $params->get('link_target', 1); $no_follow = $params->get('no_follow', 0); $enable_tooltip = $params->get('enable_tooltip', 'yes'); $tooltip_desc_words = $params->get('t_word_count_desc', 25); $tooltip_desc_images = $params->get('tooltip_desc_images', 1); $tooltip_title_words = $params->get('t_word_count_title', 25); $add_dots = !EParameter::getComponentParam(CAUTOTWEETNG, 'donot_add_dots'); if (!$rssurl) { $light_rss['error'][] = 'Invalid feed url. Please enter a valid url in the module settings.'; // Halt if no valid feed url supplied return $light_rss; } switch ($link_target) { // Open links in current or new window case 1: $link_target = '_blank'; break; case 0: $link_target = '_self'; break; default: $link_target = '_blank'; break; } $light_rss['target'] = $link_target; if ($no_follow) { $light_rss['nofollow'] = 'rel="nofollow"'; } if (!class_exists('SimplePie')) { // Include Simple Pie processor class include_once JPATH_AUTOTWEET . '/libs/SimplePie_autoloader.php'; } // Load and build the feed array $feed = new SimplePie(); $use_sp_cache = EParameter::getComponentParam(CAUTOTWEETNG, 'use_sp_cache', true); if ($use_sp_cache && is_writable(JPATH_CACHE)) { $feed->set_cache_location(JPATH_CACHE); $feed->enable_cache(true); $cache_time = intval($rsscache); $feed->set_cache_duration($cache_time); } else { $feed->enable_cache(false); } $feed->set_feed_url($rssurl); // Process the loaded feed $feed->init(); $feed->handle_content_type(); // Store any error message if (isset($feed->error)) { $light_rss['error'][] = $feed->error; } // Start building the feed meta-info (title, desc and image) // Feed title if ($feed->get_title() && $rsstitle) { $light_rss['title']['link'] = $feed->get_link(); $light_rss['title']['title'] = $feed->get_title(); } // Feed description if ($rssdesc) { $light_rss['description'] = $feed->get_description(); } // Feed image if ($rssimage && $feed->get_image_url()) { $light_rss['image']['url'] = $feed->get_image_url(); $light_rss['image']['title'] = $feed->get_image_title(); } // End feed meta-info // Start processing feed items // If there are items in the feed if ($feed->get_item_quantity()) { // Start looping through the feed items $light_rss_item = 0; // Item counter for array indexing in the loop foreach ($feed->get_items(0, $rssitems) as $currItem) { // Item title $item_title = trim($currItem->get_title()); // Item title word limit check if ($rssitemtitle_words) { $item_titles = explode(' ', $item_title); $count = count($item_titles); if ($count > $rssitemtitle_words) { $item_title = ''; for ($i = 0; $i < $rssitemtitle_words; $i++) { $item_title .= ' ' . $item_titles[$i]; } if ($add_dots) { $item_title .= '...'; } } } // Item Title $light_rss['items'][$light_rss_item]['title'] = $item_title; $light_rss['items'][$light_rss_item]['link'] = $currItem->get_permalink(); // Item description if ($rssitemdesc) { $desc = trim($currItem->get_description()); if (!$rssitemdesc_images) { // Strip image tags $desc = preg_replace("/<img[^>]+\\>/i", "", $desc); } // Item description word limit check if ($rssitemdesc_words) { $texts = explode(' ', $desc); $count = count($texts); if ($count > $rssitemdesc_words) { $desc = ''; for ($i = 0; $i < $rssitemdesc_words; $i++) { // Build words $desc .= ' ' . $texts[$i]; } if ($add_dots) { $desc .= '...'; } } } // Item Description $light_rss['items'][$light_rss_item]['description'] = $desc; } // Tooltip text if ($enable_tooltip == 'yes') { // Tooltip item title $t_item_title = trim($currItem->get_title()); // Tooltip title word limit check if ($tooltip_title_words) { $t_item_titles = explode(' ', $t_item_title); $count = count($t_item_titles); if ($count > $tooltip_title_words) { $tooltip_title = ''; for ($i = 0; $i < $tooltip_title_words; $i++) { $tooltip_title .= ' ' . $t_item_titles[$i]; } if ($add_dots) { $tooltip_title .= '...'; } } else { $tooltip_title = $t_item_title; } } else { $tooltip_title = $t_item_title; } // Replace new line characters in tooltip title, important! $tooltip_title = preg_replace("/(\r\n|\n|\r)/", " ", $tooltip_title); // Format text for tooltip $tooltip_title = htmlspecialchars(html_entity_decode($tooltip_title), ENT_QUOTES); // Tooltip Title $light_rss['items'][$light_rss_item]['tooltip']['title'] = $tooltip_title; // Tooltip item description $text = trim($currItem->get_description()); if (!$tooltip_desc_images) { $text = preg_replace("/<img[^>]+\\>/i", "", $text); } // Tooltip desc word limit check if ($tooltip_desc_words) { $texts = explode(' ', $text); $count = count($texts); if ($count > $tooltip_desc_words) { $text = ''; for ($i = 0; $i < $tooltip_desc_words; $i++) { $text .= ' ' . $texts[$i]; } if ($add_dots) { $text .= '...'; } } } // Replace new line characters in tooltip, important! $text = preg_replace("/(\r\n|\n|\r)/", " ", $text); // Format text for tooltip $text = htmlspecialchars(html_entity_decode($text), ENT_QUOTES); // Tooltip Body $light_rss['items'][$light_rss_item]['tooltip']['description'] = $text; } else { // Blank $light_rss['items'][$light_rss_item]['tooltip'] = array(); } // Increment item counter $light_rss_item++; } } // End item quantity check if statement // Return the feed data structure for the template return $light_rss; }