Allows you to override SimplePie's output to match that of your webpage.
This is useful for times when your webpages are not being served as
UTF-8. This setting will be obeyed by {@see \handle_content_type()}, and
is similar to {@see \set_input_encoding()}.
It should be noted, however, that not all character encodings can support
all characters. If your page is being served as ISO-8859-1 and you try
to display a Japanese feed, you'll likely see garbled characters.
Because of this, it is highly recommended to ensure that your webpages
are served as UTF-8.
The number of supported character encodings depends on whether your web
host supports {@link http://php.net/mbstring mbstring},
{@link http://php.net/iconv iconv}, or both. See
{@link http://simplepie.org/wiki/faq/Supported_Character_Encodings} for
more information.
public set_output_encoding ( string $encoding = 'UTF-8' ) | ||
$encoding | string |
/** * Display a wildcard in the back end * * @return string */ public function generate() { if (TL_MODE == 'BE') { /** @var \BackendTemplate|object $objTemplate */ $objTemplate = new \BackendTemplate('be_wildcard'); $objTemplate->wildcard = '### ' . utf8_strtoupper($GLOBALS['TL_LANG']['FMD']['rss_reader'][0]) . ' ###'; $objTemplate->title = $this->headline; $objTemplate->id = $this->id; $objTemplate->link = $this->name; $objTemplate->href = '' . $GLOBALS['TL_CONFIG']['backendPath'] . '/main.php?do=themes&table=tl_module&act=edit&id=' . $this->id; return $objTemplate->parse(); } $this->objFeed = new \SimplePie(); $arrUrls = trimsplit('[\\n\\t ]', trim($this->rss_feed)); if (count($arrUrls) > 1) { $this->objFeed->set_feed_url($arrUrls); } else { $this->objFeed->set_feed_url($arrUrls[0]); } $this->objFeed->set_output_encoding(\Config::get('characterSet')); $this->objFeed->set_cache_location(TL_ROOT . '/system/tmp'); $this->objFeed->enable_cache(false); if ($this->rss_cache > 0) { $this->objFeed->enable_cache(true); $this->objFeed->set_cache_duration($this->rss_cache); } if (!$this->objFeed->init()) { $this->log('Error importing RSS feed "' . $this->rss_feed . '"', __METHOD__, TL_ERROR); return ''; } $this->objFeed->handle_content_type(); return parent::generate(); }
public function addSystemMessages() { if ($GLOBALS['TL_CONFIG']['be_rss_src'] == '') { return ''; } $feed = new \SimplePie(); $feed->set_feed_url(html_entity_decode($GLOBALS['TL_CONFIG']['be_rss_src'])); $feed->set_output_encoding(\Config::get('characterSet')); $feed->set_cache_location(TL_ROOT . '/system/tmp'); $feed->enable_cache(true); if (!$feed->init()) { $this->log('Error importing RSS feed "' . $this->rss_feed . '"', __METHOD__, TL_ERROR); return ''; } $items = $feed->get_items(0, $GLOBALS['TL_CONFIG']['be_rss_max'] ? $GLOBALS['TL_CONFIG']['be_rss_max'] : 3); $output = ''; if ($items) { $user = \BackendUser::getInstance(); foreach ($items as $item) { $template = new \BackendTemplate('be_rss_item'); $template->title = $item->get_title(); $template->link = $item->get_link(); $template->content = $item->get_content(); $template->date = $item->get_date($GLOBALS['TL_CONFIG']['datimFormat']); $template->class = $item->get_date('U') > $user->lastLogin ? 'new' : 'message'; $output .= $template->parse(); } } $template = new \BackendTemplate('be_rss'); $template->headline = $GLOBALS['TL_CONFIG']['be_rss_headline']; $template->items = $output; return $template->parse(); }
function flickr_photos($feed_url, $max_items = 10) { //$items = combine_feeds( array($feed_url), $max_items, '~', false); $feed = new SimplePie(); $feed->set_feed_url($feed_url); $feed->set_cache_location(ABSPATH . '/cache'); $feed->set_output_encoding('ISO-8859-1'); $feed->init(); $html = ''; if ($feed->data) { foreach ($feed->get_items(0, $max_items) as $item) { $image = $item->get_description(); $image = substr($image, strpos($image, 'src=') + 4); // '<img') + 10); $image = trim(substr($image, 0, strpos($image, '.jpg') + 4)); // , "\" width"))); $healthy = array("%3A", "%2F", '"', 'm.jpg'); $yummy = array(":", "/", '', 's.jpg'); $image = str_replace($healthy, $yummy, $image); //$image = str_replace('m.jpg', 's.jpg', $image); $html .= '<a href="' . $item->get_permalink() . '">'; $html .= '<img src="' . $image . '" alt="[flickr photo: ' . $item->get_title() . ']" title="' . $item->get_title() . '" />'; $html .= "</a>\n"; } } return $html; }
public function html() { // Get settings $settings = $this->config; // Define Simplepie $feed = new \SimplePie(); $feed->set_feed_url($settings['feed']); $feed->enable_cache($settings['enable_cache']); $feed->set_cache_location(cache_path()); $feed->set_cache_duration(60 * 60 * 12); $feed->set_output_encoding($settings['charset']); $feed->init(); $title = $settings['title']; $data = []; foreach ($feed->get_items($settings['offset'], $settings['limit']) as $key => $item) { $data[$key]['title'] = $item->get_title(); $data[$key]['permalink'] = $item->get_permalink(); $data[$key]['date'] = $item->get_date(); $data[$key]['updated_date'] = $item->get_updated_date(); $data[$key]['author'] = $item->get_author(); $data[$key]['category'] = $item->get_category(); $data[$key]['description'] = $item->get_description(); $data[$key]['content'] = $item->get_content(); } return $this->view('rssfeed', compact('title', 'data')); }
function fetchRss($url) { $feed = new SimplePie(); $feed->set_feed_url($url); $feed->set_output_encoding("UTF-8"); $feed->enable_order_by_date(false); $feed->set_cache_location(Configure::read('Rss.cache_path')); $feed->init(); return $feed->get_items(); }
function __construct($feed_url = null) { global $CFG; // Use the Moodle class for http requests $this->file_class = 'moodle_simplepie_file'; // Use sensible cache directory $cachedir = $CFG->dataroot . '/cache/simplepie/'; if (!file_exists($cachedir)) { mkdir($cachedir, 0777, true); } parent::__construct($feed_url, $cachedir); parent::set_output_encoding('UTF-8'); }
protected function fetchSite($site) { $feed = new \SimplePie(); $feed->force_feed(true); $feed->set_item_limit(20); $feed->set_feed_url($site); $feed->enable_cache(false); $feed->set_output_encoding('utf-8'); $feed->init(); foreach ($feed->get_items() as $item) { $this->outputItem(['site' => $site, 'title' => $item->get_title(), 'link' => $item->get_permalink(), 'date' => new \Carbon\Carbon($item->get_date()), 'content' => $item->get_content()]); } }
public function setCron() { $getUnique = $this->getAllFeedUrls(); if (!empty($getUnique)) { foreach ($getUnique as $feedTableData) { $feed_user_title = $feedTableData->feed_user_title; $feed_url = $feedTableData->feed_admin_url; $usercat = $feedTableData->feed_user_category; $feed_filter_type = $feedTableData->feed_filter_type; $favicon_icon = $feedTableData->feed_favicon; $feed = new SimplePie(); $feed->set_feed_url($feed_url); $feed->set_cache_location(APPPATH . '/cache'); $feed->set_output_encoding('ISO-8859-1'); $feed->init(); $feed->handle_content_type(); // Language $lang = $feed->get_language(); $language = isset($lang) ? $lang : 'en-us'; if ($feed->get_type() & SIMPLEPIE_TYPE_NONE) { $feed_type = 'Unknown'; } elseif ($feed->get_type() & SIMPLEPIE_TYPE_RSS_ALL) { $feed_type = 'RSS'; } elseif ($feed->get_type() & SIMPLEPIE_TYPE_ATOM_ALL) { $feed_type = 'Atom'; } elseif ($feed->get_type() & SIMPLEPIE_TYPE_ALL) { $feed_type = 'Supported'; } // Author if ($author = $feed->get_author()) { $feedAuthor = $author->get_name(); } else { $feedAuthor = ''; } if ($feed->error()) { die; } else { $feed_image_link = $feed->get_image_link(); $feed_image_url = $feed->get_image_url(); $feed_image_title = $feed->get_image_title(); $this->addFeeds($feed->get_items(0, 500), $feed_image_link, $feed_image_url, $feed_image_title, $usercat, $feed_url, $language, $feed_type, $feedAuthor, $feed_filter_type, $feed_user_title, $favicon_icon); } } } }
function powerpress_get_news($feed_url, $limit = 10) { include_once ABSPATH . WPINC . '/feed.php'; $rss = fetch_feed($feed_url); // If feed doesn't work... if (is_wp_error($rss)) { require_once ABSPATH . WPINC . '/class-feed.php'; // Try fetching the feed using CURL directly... $content = powerpress_remote_fopen($feed_url, false, array(), 3, false, true); if (!$content) { return false; } // Load the content in a fetch_feed object... $rss = new SimplePie(); $rss->set_sanitize_class('WP_SimplePie_Sanitize_KSES'); // We must manually overwrite $feed->sanitize because SimplePie's // constructor sets it before we have a chance to set the sanitization class $rss->sanitize = new WP_SimplePie_Sanitize_KSES(); $rss->set_cache_class('WP_Feed_Cache'); $rss->set_file_class('WP_SimplePie_File'); $rss->set_raw_data($content); $rss->set_cache_duration(apply_filters('wp_feed_cache_transient_lifetime', 12 * HOUR_IN_SECONDS, $feed_url)); do_action_ref_array('wp_feed_options', array(&$rss, $feed_url)); $rss->init(); $rss->set_output_encoding(get_option('blog_charset')); $rss->handle_content_type(); if ($rss->error()) { return false; } } $rss_items = $rss->get_items(0, $rss->get_item_quantity($limit)); // If the feed was erroneously if (!$rss_items) { $md5 = md5($this->feed); delete_transient('feed_' . $md5); delete_transient('feed_mod_' . $md5); $rss->__destruct(); unset($rss); $rss = fetch_feed($this->feed); $rss_items = $rss->get_items(0, $rss->get_item_quantity($num)); $rss->__destruct(); unset($rss); } return $rss_items; }
public function check() { $feed = new SimplePie(); $feed->set_feed_url('http://zeenews.india.com/rss/india-national-news.xml'); $feed->set_cache_location(APPPATH . '/cache'); $feed->set_output_encoding('ISO-8859-1'); $feed->init(); $feed->handle_content_type(); // $i = $feed->get_items(0, 500); $allFeeds = array(); foreach ($feed->get_items(0, 100) as $item) { if ($author = $item->get_author()) { $email = $author->get_email(); $name = $author->get_name(); if ($name != '') { $authorname = $name; } else { $authorname = $email; } } $date = $item->get_date(); $singleFeed = array('author' => $item->get_author(), 'authoremail' => $item->get_author(), 'categories' => $item->get_categories(), 'copyright' => $item->get_copyright(), 'content' => $item->get_content(), 'date' => $item->get_date("d.m.Y H:i"), 'description' => $item->get_description(), 'id' => $item->get_id(), 'latitude' => $item->get_latitude(), 'longitude' => $item->get_longitude(), 'permalink' => $item->get_permalink(), 'title' => $item->get_title()); pri($item); array_push($allFeeds, $singleFeed); echo $date . '$date<br>'; } die; }
public static function getFeedData($params) { $rssurl = str_replace("\n", ",", JString::trim($params->get('rssurl', ''))); $feedTimeout = (int) $params->get('feedTimeout', 10); $rssdateorder = (int) $params->get('rssdateorder', 1); $dformat = $params->get('dformat', '%d %b %Y %H:%M %P'); $rssperfeed = (int) $params->get('rssperfeed', 3); $textfilter = JString::trim($params->get('textfilter', '')); $pagination = (int) $params->get('pagination', 0); $totalfeeds = (int) $params->get('rssitems', 5); $filtermode = (int) $params->get('filtermode', 0); $showfeedinfo = (int) $params->get('showfeedinfo', 1); $input_encoding = JString::trim($params->get('inputencoding', '')); $showimage = (int) $params->get('rssimage', 1); $cacheTime = (int) $params->get('feedcache', 15) * 60; //minutes $orderBy = $params->get('orderby', 'date'); $tmzone = (int) $params->get('tmzone', 0) ? true : false; $cachePath = JPATH_SITE . DS . 'cache' . DS . 'mod_we_ufeed_display'; $start = $end = 0; if ($pagination) { $pagination_items = (int) $params->get('paginationitems', 5); $current_limit = modFeedShowHelper::getPage($params->get('mid', 0)); $start = ($current_limit - 1) * $pagination_items; $end = $current_limit * $pagination_items; } #Get clean array $rss_urls = @array_filter(explode(",", $rssurl)); #If only 1 link, use totalfeeds for total limit if (count($rss_urls) == 1) { $rssperfeed = $totalfeeds; } # Intilize RSS Doc if (!class_exists('SimplePie')) { jimport('simplepie.simplepie'); } //Parser Code $simplepie = new SimplePie(); $simplepie->set_cache_location($cachePath); $simplepie->set_cache_duration($cacheTime); $simplepie->set_stupidly_fast(true); $simplepie->force_feed(true); //$simplepie->force_fsockopen(false); //gives priority to CURL if is installed $simplepie->set_timeout($feedTimeout); $simplepie->set_item_limit($rssperfeed); $simplepie->enable_order_by_date(false); if ($input_encoding) { $simplepie->set_input_encoding($input_encoding); $simplepie->set_output_encoding('UTF-8'); } $simplepie->set_feed_url($rss_urls); $simplepie->init(); $rssTotalItems = (int) $simplepie->get_item_quantity($totalfeeds); if ((int) $params->get('debug', 0)) { echo "<h3>Total RSS Items:" . $rssTotalItems . "</h3>"; echo print_r($simplepie, true); #debug } if (get_class($simplepie) != 'SimplePie' || !$rssTotalItems) { return array("rsstotalitems" => 0, 'items' => false); } $feedItems = array(); #store all feeds items $counter = 1; foreach ($simplepie->get_items($start, $end) as $key => $feed) { #Word Filter if (!empty($textfilter) && $filtermode != 0) { $filter = modFeedShowHelper::filterItems($feed, $textfilter); #Include #Exclude if ($filtermode == 1 && !$filter || $filtermode == 2 && $filter) { $rssTotalItems--; continue; #Include } } $FeedValues[$key] = new stdClass(); # channel header and link $channel = $feed->get_feed(); $FeedValues[$key]->FeedTitle = $channel->get_title(); $FeedValues[$key]->FeedLink = $channel->get_link(); if ($showfeedinfo) { $FeedValues[$key]->FeedFavicon = 'http://g.etfv.co/' . urlencode($FeedValues[$key]->FeedLink); } $FeedValues[$key]->FeedDescription = $channel->get_description(); $FeedValues[$key]->FeedLogo = $channel->get_image_url(); #Item $FeedValues[$key]->ItemTitle = $feed->get_title(); $feeDateUNIX = $feed->get_date('U'); if ($feeDateUNIX < 1) { $feeDateUNIX = strtotime(trim(str_replace(",", "", $feed->get_date('')))); } $FeedValues[$key]->ItemDate = WEJM16 ? JHTML::_('date', $feeDateUNIX, $dformat, $tmzone) : JHtml::date($feeDateUNIX, $dformat, $tmzone); $FeedValues[$key]->ItemLink = $feed->get_link(); //$feed->get_permalink(); $FeedValues[$key]->ItemText = $feed->get_description(); $FeedValues[$key]->ItemFulltext = $feed->get_content(); $FeedValues[$key]->ItemEnclosure = $feed->get_enclosure(); $FeedValues[$key]->ItemEnclosures = $feed->get_enclosures(); if ($showimage) { $FeedValues[$key]->ItemImage = ""; } //for next version if ($orderBy == 'title') { $idx = str_replace(array(':', '-', '(', ')', '+', '*', ' '), '', JString::strtolower(strip_tags($FeedValues[$key]->ItemTitle))); //ORDER BY TITLE } else { $idx = $feeDateUNIX; //Order By date } if (isset($feedItems[$idx])) { $idx .= $counter; } #unique idx $feedItems[$idx] = $FeedValues[$key]; $counter++; } if ($rssdateorder == 1) { krsort($feedItems); } elseif ($rssdateorder == 2) { ksort($feedItems); } elseif ($rssdateorder == 3) { shuffle($feedItems); } if ((int) $params->get('debug', 0)) { echo "<p>Total RSS in Array:" . count($feedItems) . "</p>"; } return array("rsstotalitems" => $rssTotalItems, 'items' => $feedItems); }
function fetchFeeds($opt = null) { global $serendipity; set_time_limit(360); ignore_user_abort(true); $_SESSION['serendipityRightPublish'] = true; $serendipity['noautodiscovery'] = true; $this->setupDB(); $feeds = $this->getFeeds($opt); $engine = $this->get_config('engine', 'onyx'); if ($engine == 'onyx') { require_once (defined('S9Y_PEAR_PATH') ? S9Y_PEAR_PATH : S9Y_INCLUDE_PATH . 'bundled-libs/') . 'Onyx/RSS.php'; } elseif ($engine == 'magpierss') { // CLSC: NEW "MagpieRSS" include require_once dirname(__FILE__) . '/magpierss/rss_fetch.inc'; } elseif ($engine == 'simplepie') { //hwa: NEW "SimplePie" include require_once dirname(__FILE__) . '/simplepie/simplepie.inc'; } $cache_authors = array(); $cache_entries = array(); $cache_md5 = array(); $sql_cache_authors = serendipity_db_Query("SELECT authorid, realname\n FROM {$serendipity['dbPrefix']}authors"); if (is_array($sql_cache_authors)) { foreach ($sql_cache_authors as $idx => $author) { $cache_authors[$author['realname']] = $author['authorid']; } } if ($this->debug) { printf("DEBUG: cache_authors['realname'] = authorid has %d entries\n", count($cache_authors)); } if ($opt['store_seperate']) { $sql_cache_entries = serendipity_db_query("SELECT e.feedid, e.id, e.entrydate, e.entrytitle\n FROM {$serendipity['dbPrefix']}aggregator_feedlist AS e"); if (is_array($sql_cache_entries)) { foreach ($sql_cache_entries as $idx => $entry) { $cache_entries[$entry['entrytitle']][$entry['feedid']][$entry['entrydate']] = $entry['id']; } } } else { $sql_cache_entries = serendipity_db_query("SELECT e.id, e.timestamp, e.authorid, e.title, ep.value\n FROM {$serendipity['dbPrefix']}entries AS e,\n {$serendipity['dbPrefix']}entryproperties AS ep\n WHERE e.id = ep.entryid\n AND ep.property = 'ep_aggregator_feed'"); if (is_array($sql_cache_entries)) { foreach ($sql_cache_entries as $idx => $entry) { $cache_entries[$entry['title']][$entry['authorid']][$entry['timestamp']] = $entry['id']; } } } if ($this->debug) { printf("DEBUG: cache_entries['title']['authorid']['timestamp'] = entryid has %d entries.\n", count($cache_entries)); } $sql_cache_md5 = serendipity_db_query("SELECT entryid, md5, timestamp\n FROM {$serendipity['dbPrefix']}aggregator_md5"); if (is_array($sql_cache_md5)) { foreach ($sql_cache_md5 as $idx => $entry) { $cache_md5[$entry['md5']]['entryid'] = $entry['entryid']; $cache_md5[$entry['md5']]['timestamp'] = $entry['timestamp']; } } if ($this->debug) { printf("DEBUG: cache_md5['md5'] = entryid has %d entries.\n", count($cache_md5)); } foreach ($feeds as $feed) { if (!$opt['store_seperate']) { printf("Read %s.\n", $feed['feedurl']); } flush(); $feed_authorid = $cache_authors[$feed['feedname']]; if (empty($feed_authorid)) { $feed_authorid = 0; } if ($this->debug) { printf("DEBUG: Current authorid = %d\n", $feed_authorid); } $stack = array(); if ($engine == 'onyx') { if (empty($feed['charset'])) { $this->checkCharset($feed); } # test multiple likely charsets $charsets = array($feed['charset'], "ISO-8859-1", "utf-8"); $retry = false; foreach ($charsets as $ch) { if ($retry) { printf("DEBUG: Retry with charset %s instead of %s\n", $ch, $feed['charset']); } $retry = true; $c = new Onyx_RSS($ch); # does it parse? if so, all is fine... if ($c->parse($feed['feedurl'])) { break; } } while ($item = $c->getNextItem()) { /* Okay this is where things get tricky. Everybody * encodes their information differently. For now I'm going to focus on * s9y weblogs. */ $fake_timestamp = false; $date = $this->parseDate($item['pubdate']); if ($this->debug) { printf("DEBUG: pubDate %s = %s\n", $item['pubdate'], $date); } if ($date == -1) { // Fallback to try for dc:date $date = $this->parseDate($item['dc:date']); if ($this->debug) { printf("DEBUG: falling back to dc:date % s= %s\n", $item['dc:date'], $date); } } if ($date == -1) { // Couldn't figure out the date string. Set it to "now" and hope that the md5hash will get it. $date = time(); $fake_timestamp = true; if ($this->debug) { printf("DEBUG: falling back to time() = %s\n", $date); } } if (empty($item['title'])) { if ($this->debug) { printf("DEBUG: skip item: title was empty for %s\n", print_r($item, true)); } continue; } $this->decode($c->rss['encoding'], $item); $item['date'] = $date; $stack[] = $item; } } elseif ($engine == 'magpierss') { // ---------------------------------------------------------- // CLSC: New MagpieRSS code start // ---------------------------------------------------------- $rss = fetch_rss($feed['feedurl']); foreach ($rss->items as $item) { $fake_timestamp = false; $date = $item['pubdate']; if ($this->debug) { printf("DEBUG: pubdate = %s\n", $item['pubdate'], $date); } // ---------------------------------------------------------- // CLSC: Try a few different types of timestamp fields // So that we might get lucky even with non-standard feeds // ---------------------------------------------------------- if ($date == "") { // CLSC: magpie syntax for nested fields $date = $item['dc']['date']; if ($this->debug) { printf("DEBUG: falling back to [dc][date] = %s\n", $item['dc:date'], $date); } } if ($date == "") { $date = $item['modified']; if ($this->debug) { printf("DEBUG: falling back to modified = %s\n", $item['modified'], $date); } } if ($date == "") { $date = $item['PubDate']; if ($this->debug) { printf("DEBUG: falling back PubDate = %s\n", $item['PubDate'], $date); } } if ($date == "") { $date = $item['created']; if ($this->debug) { printf("DEBUG: falling back to created = %s\n", $item['created'], $date); } } if ($date == "") { // CLSC: not proper magpie syntax but still catches some $date = $item['dc:date']; if ($this->debug) { printf("DEBUG: falling back to dc:date = %s\n", $item['dc:date'], $date); } } if ($date == "") { $date = $item['updated']; if ($this->debug) { printf("DEBUG: falling back to updated = %s\n", $item['updated'], $date); } } if ($date == "") { $date = $item['published']; if ($this->debug) { printf("DEBUG: falling back to published = %s\n", $item['published'], $date); } } if ($date == "") { // ---------------------------------------------------------- // CLSC: If none of the above managed to identify a date: // Set date to "now" and hope that the md5hash will get it. // ---------------------------------------------------------- $date = time(); $fake_timestamp = true; if ($this->debug) { printf("DEBUG: falling back to time() = %s\n", $date); } } // CLSC: if date is set to "now" parseDate can't parse it. if ($fake_timestamp != true) { $date = $this->parseDate($date); } if ($item['title'] == "") { if ($this->debug) { printf("DEBUG: skip item: title was empty for %s\n", print_r($item, true)); } continue; } $item['date'] = $date; $stack[] = $item; // ---------------------------------------------------------- // CLSC: New MagpieRSS code end // ---------------------------------------------------------- } } elseif ($engine == 'simplepie') { // hwa: new SimplePie code ; lifted from the SimplePie demo $simplefeed = new SimplePie(); $simplefeed->cache = false; $simplefeed->set_feed_url($feed['feedurl']); // Initialize the whole SimplePie object. Read the feed, process it, parse it, cache it, and // all that other good stuff. The feed's information will not be available to SimplePie before // this is called. $success = $simplefeed->init(); // We'll make sure that the right content type and character encoding gets set automatically. // This function will grab the proper character encoding, as well as set the content type to text/html. $simplefeed->set_output_encoding(LANG_CHARSET); $simplefeed->handle_content_type(); $item['new_feedicon'] = $simplefeed->get_favicon(); // error handling if ($simplefeed->error()) { if (!$opt['store_seperate']) { printf('<p><b>ERROR:</b> ' . (function_exists('serendipity_specialchars') ? serendipity_specialchars($simplefeed->error()) : htmlspecialchars($simplefeed->error(), ENT_COMPAT, LANG_CHARSET)) . "</p>\r\n"); } } if ($success) { foreach ($simplefeed->get_items() as $simpleitem) { // map SimplePie items to s9y items $item['title'] = $simpleitem->get_title(); $item['date'] = $simpleitem->get_date('U'); $item['pubdate'] = $simpleitem->get_date('U'); $item['description'] = $simpleitem->get_description(); $item['content'] = $simpleitem->get_content(); $item['link'] = $simpleitem->get_permalink(); $item['author'] = $simpleitem->get_author(); //if ($this->debug) { // printf("DEBUG: SimplePie item: author: $item['author'], title: $item['title'], date: $item['date']\n"); //} $stack[] = $item; } } else { if (!$opt['store_seperate']) { printf('<p><b>ERROR:</b> ' . print_r($success, true) . "</p>\r\n"); } } } while (list($key, $item) = each($stack)) { if ($opt['store_seperate']) { $ep_id = $cache_entries[$item['title']][$feed['feedid']][$item['date']]; if ($this->debug) { printf("DEBUG: lookup cache_entries[%s][%s][%s] finds %s.\n", $item['title'], $feed['feedid'], $item['date'], empty($ep_id) ? "nothing" : $ep_id); } } else { $ep_id = $cache_entries[$item['title']][$feed_authorid][$item['date']]; if ($this->debug) { printf("DEBUG: lookup cache_entries[%s][%s][%s] finds %s.\n", $item['title'], $feed_authorid, $item['date'], empty($ep_id) ? "nothing" : $ep_id); } } if (!empty($ep_id) and serendipity_db_bool($this->get_config('ignore_updates'))) { if ($this->debug) { printf("DEBUG: entry %s is known and ignore_updates is set.\n", $ep_id); } continue; } # NOTE: If $ep_id is NULL or EMPTY, it means that an entry with this title does not # yet exist. Later on we check if a similar entry with the body exists and skip # updates in this case. Else it means that the new entry needs to be inserted # as a new one. # The entry is probably new? $entry = array('id' => $ep_id, 'title' => $item['title'], 'timestamp' => $item['date'], 'extended' => '', 'isdraft' => serendipity_db_bool($this->get_config('publishflag')) ? 'false' : 'true', 'allow_comments' => serendipity_db_bool($this->get_config('allow_comments')) ? 'true' : 'false', 'categories' => $feed['categoryids'], 'author' => $feed['feedname'], 'authorid' => $feed_authorid); // ---------------------------------------------------------- // CLSC: Added a few flavours if ($item['content:encoded']) { $entry['body'] = $item['content:encoded']; } elseif ($item['description']) { $entry['body'] = $item['description']; } elseif ($item['content']['encoded']) { $entry['body'] = $item['content']['encoded']; } elseif ($item['atom_content']) { $entry['body'] = $item['atom_content']; } elseif ($item['content']) { $entry['body'] = $item['content']; } $md5hash = md5($feed_authorid . $item['title'] . $entry['body']); # Check 1: Have we seen this MD5? if ($this->debug) { printf("DEBUG: lookup cache_md5[%s] finds %s.\n", $md5hash, empty($cache_md5[$md5hash]) ? "nothing" : $cache_md5[$md5hash]['entryid']); } # If we have this md5, title and body for this article # are unchanged. We do not need to do anything. if (isset($cache_md5[$md5hash])) { continue; } # Check 2 (conditional: expire enabled?): # Is this article too old? if ($this->get_config('expire') > 0) { $expire = time() - 86400 * $this->get_config('expire'); if ($item['date'] < $expire) { if ($this->debug) { printf("DEBUG: '%s' is too old (%s < %s).\n", $item['title'], $item['date'], $expire); } continue; } } # Check 3: Does this article match our expressions? if (!empty($feed['match_expression'])) { $expressions = explode("~", $feed['match_expression']); $match = 0; foreach ($expressions as $expression) { $expression = ltrim(rtrim($expression)); if (preg_match("~{$expression}~imsU", $entry['title'] . $entry['body'])) { $match = 1; } } if ($match == 0) { continue; } } $feed['articleurl'] = $item['link']; if ($item['author']) { $feed['author'] = $item['author']; } elseif ($item['dc:creator']) { $feed['author'] = $item['dc:creator']; } // Store as property // Plugins might need this. $serendipity['POST']['properties'] = array('fake' => 'fake'); $markups = explode('^', $this->get_config('markup')); if (is_array($markups)) { foreach ($markups as $markup) { $serendipity['POST']['properties']['disable_markups'][] = $markup; } } if ($opt['store_seperate']) { if ($entry['id'] > 0) { serendipity_db_query("UPDATE {$serendipity['dbPrefix']}aggregator_feedlist \n SET feedid = '" . $feed['feedid'] . "',\n categoryid = '" . $feed['categoryids'][0] . "',\n entrydate = '" . serendipity_db_escape_string($entry['timestamp']) . "',\n entrytitle = '" . serendipity_db_escape_string($entry['title']) . "',\n entrybody = '" . serendipity_db_escape_string($entry['body']) . "',\n entryurl = '" . serendipity_db_escape_string($item['link']) . "'\n WHERE id = " . $entry['id']); $entryid = $entry['id']; } else { serendipity_db_query("INSERT INTO {$serendipity['dbPrefix']}aggregator_feedlist (\n feedid,\n categoryid,\n entrydate,\n entrytitle,\n entrybody,\n entryurl\n ) VALUES (\n '" . $feed['feedid'] . "',\n '" . $feed['categoryids'][0] . "',\n '" . serendipity_db_escape_string($entry['timestamp']) . "',\n '" . serendipity_db_escape_string($entry['title']) . "',\n '" . serendipity_db_escape_string($entry['body']) . "',\n '" . serendipity_db_escape_string($item['link']) . "'\n )"); $entryid = serendipity_db_insert_id(); } $this->feedupdate_finish($feed, $entryid); } else { $entryid = serendipity_updertEntry($entry); $this->insertProperties($entryid, $feed, $md5hash); } if (!$opt['store_seperate']) { printf(" Save '%s' as %s.\n", $item['title'], $entryid); } } if (!$opt['store_seperate']) { printf("Finish feed.\n"); } } if (!$opt['store_seperate']) { printf("Finish planetarium.\n"); } }
?> " method="post"> <b>Enter Feed Url</b><br /> <input name="url" type="text" value="<?php echo $_POST['url']; ?> " style="width:400px;"><input name="Submit" type="submit" value="Get Feed"> </form> <?php if (isset($_POST['url']) && $_POST['url'] != "") { $url = $_POST['url']; include_once 'Simple/autoloader.php'; $feed = new SimplePie(); $feed->set_feed_url($url); $feed->enable_cache(false); $feed->set_output_encoding('Windows-1252'); $feed->init(); echo "<span><h1>" . $feed->get_title() . "</h1>"; echo "<b>" . $feed->get_description() . "</b></span><hr />"; $itemCount = $feed->get_item_quantity(); $items = $feed->get_items(); foreach ($items as $item) { ?> <div><a href="<?php echo $item->get_permalink(); ?> "><?php echo $item->get_title(); ?> </a><br /> <em style="font-size:.7em;color:#666666"><?php
function update_rss_feed($link, $feed, $ignore_daemon = false, $no_cache = false, $override_url = false) { require_once "lib/simplepie/simplepie.inc"; $debug_enabled = defined('DAEMON_EXTENDED_DEBUG') || $_REQUEST['xdebug']; if ($debug_enabled) { _debug("update_rss_feed: start"); } $result = db_query($link, "SELECT id,update_interval,auth_login,\n\t\t\tfeed_url,auth_pass,cache_images,last_updated,\n\t\t\tmark_unread_on_update, owner_uid,\n\t\t\tpubsub_state\n\t\t\tFROM ttrss_feeds WHERE id = '{$feed}'"); if (db_num_rows($result) == 0) { if ($debug_enabled) { _debug("update_rss_feed: feed {$feed} NOT FOUND/SKIPPED"); } return false; } $last_updated = db_fetch_result($result, 0, "last_updated"); $owner_uid = db_fetch_result($result, 0, "owner_uid"); $mark_unread_on_update = sql_bool_to_bool(db_fetch_result($result, 0, "mark_unread_on_update")); $pubsub_state = db_fetch_result($result, 0, "pubsub_state"); db_query($link, "UPDATE ttrss_feeds SET last_update_started = NOW()\n\t\t\tWHERE id = '{$feed}'"); $auth_login = db_fetch_result($result, 0, "auth_login"); $auth_pass = db_fetch_result($result, 0, "auth_pass"); $cache_images = sql_bool_to_bool(db_fetch_result($result, 0, "cache_images")); $fetch_url = db_fetch_result($result, 0, "feed_url"); $feed = db_escape_string($feed); /* if ($auth_login && $auth_pass ){ $url_parts = array(); preg_match("/(^[^:]*):\/\/(.*)/", $fetch_url, $url_parts); if ($url_parts[1] && $url_parts[2]) { $fetch_url = $url_parts[1] . "://$auth_login:$auth_pass@" . $url_parts[2]; } } */ if ($override_url) { $fetch_url = $override_url; } if ($debug_enabled) { _debug("update_rss_feed: fetching [{$fetch_url}]..."); } // Ignore cache if new feed or manual update. $cache_age = is_null($last_updated) || $last_updated == '1970-01-01 00:00:00' ? -1 : get_feed_update_interval($link, $feed) * 60; $simplepie_cache_dir = CACHE_DIR . "/simplepie"; if (!is_dir($simplepie_cache_dir)) { mkdir($simplepie_cache_dir); } $feed_data = fetch_file_contents($fetch_url, false, $auth_login, $auth_pass, false, $no_cache ? 15 : 45); if (!$feed_data) { global $fetch_last_error; if ($debug_enabled) { _debug("update_rss_feed: unable to fetch: {$fetch_last_error}"); } $error_escaped = db_escape_string($fetch_last_error); db_query($link, "UPDATE ttrss_feeds SET last_error = '{$error_escaped}',\n\t\t\t\t\tlast_updated = NOW() WHERE id = '{$feed}'"); return; } $pluginhost = new PluginHost($link); $pluginhost->set_debug($debug_enabled); $user_plugins = get_pref($link, "_ENABLED_PLUGINS", $owner_uid); $pluginhost->load(PLUGINS, $pluginhost::KIND_ALL); $pluginhost->load($user_plugins, $pluginhost::KIND_USER, $owner_uid); $pluginhost->load_data(); foreach ($pluginhost->get_hooks($pluginhost::HOOK_FEED_FETCHED) as $plugin) { $feed_data = $plugin->hook_feed_fetched($feed_data); } if ($debug_enabled) { _debug("update_rss_feed: fetch done, parsing..."); } $rss = new SimplePie(); $rss->set_sanitize_class("SanitizeDummy"); // simplepie ignores the above and creates default sanitizer anyway, // so let's override it... $rss->sanitize = new SanitizeDummy(); $rss->set_output_encoding('UTF-8'); $rss->set_raw_data($feed_data); if ($debug_enabled) { _debug("feed update interval (sec): " . get_feed_update_interval($link, $feed) * 60); } $rss->enable_cache(!$no_cache); if (!$no_cache) { $rss->set_cache_location($simplepie_cache_dir); $rss->set_cache_duration($cache_age); } @$rss->init(); // print_r($rss); $feed = db_escape_string($feed); if (!$rss->error()) { // We use local pluginhost here because we need to load different per-user feed plugins $pluginhost->run_hooks($pluginhost::HOOK_FEED_PARSED, "hook_feed_parsed", $rss); if ($debug_enabled) { _debug("update_rss_feed: processing feed data..."); } // db_query($link, "BEGIN"); if (DB_TYPE == "pgsql") { $favicon_interval_qpart = "favicon_last_checked < NOW() - INTERVAL '12 hour'"; } else { $favicon_interval_qpart = "favicon_last_checked < DATE_SUB(NOW(), INTERVAL 12 HOUR)"; } $result = db_query($link, "SELECT title,site_url,owner_uid,\n\t\t\t\t(favicon_last_checked IS NULL OR {$favicon_interval_qpart}) AS\n\t\t\t\t\t\tfavicon_needs_check\n\t\t\t\tFROM ttrss_feeds WHERE id = '{$feed}'"); $registered_title = db_fetch_result($result, 0, "title"); $orig_site_url = db_fetch_result($result, 0, "site_url"); $favicon_needs_check = sql_bool_to_bool(db_fetch_result($result, 0, "favicon_needs_check")); $owner_uid = db_fetch_result($result, 0, "owner_uid"); $site_url = db_escape_string(mb_substr(rewrite_relative_url($fetch_url, $rss->get_link()), 0, 245)); if ($debug_enabled) { _debug("update_rss_feed: checking favicon..."); } if ($favicon_needs_check) { check_feed_favicon($site_url, $feed, $link); db_query($link, "UPDATE ttrss_feeds SET favicon_last_checked = NOW()\n\t\t\t\t\tWHERE id = '{$feed}'"); } if (!$registered_title || $registered_title == "[Unknown]") { $feed_title = db_escape_string($rss->get_title()); if ($debug_enabled) { _debug("update_rss_feed: registering title: {$feed_title}"); } db_query($link, "UPDATE ttrss_feeds SET\n\t\t\t\t\ttitle = '{$feed_title}' WHERE id = '{$feed}'"); } if ($site_url && $orig_site_url != $site_url) { db_query($link, "UPDATE ttrss_feeds SET\n\t\t\t\t\tsite_url = '{$site_url}' WHERE id = '{$feed}'"); } if ($debug_enabled) { _debug("update_rss_feed: loading filters & labels..."); } $filters = load_filters($link, $feed, $owner_uid); $labels = get_all_labels($link, $owner_uid); if ($debug_enabled) { //print_r($filters); _debug("update_rss_feed: " . count($filters) . " filters loaded."); } $items = $rss->get_items(); if (!is_array($items)) { if ($debug_enabled) { _debug("update_rss_feed: no articles found."); } db_query($link, "UPDATE ttrss_feeds\n\t\t\t\t\tSET last_updated = NOW(), last_error = '' WHERE id = '{$feed}'"); return; // no articles } if ($pubsub_state != 2 && PUBSUBHUBBUB_ENABLED) { if ($debug_enabled) { _debug("update_rss_feed: checking for PUSH hub..."); } $feed_hub_url = false; $links = $rss->get_links('hub'); if ($links && is_array($links)) { foreach ($links as $l) { $feed_hub_url = $l; break; } } if ($debug_enabled) { _debug("update_rss_feed: feed hub url: {$feed_hub_url}"); } if ($feed_hub_url && function_exists('curl_init') && !ini_get("open_basedir")) { require_once 'lib/pubsubhubbub/subscriber.php'; $callback_url = get_self_url_prefix() . "/public.php?op=pubsub&id={$feed}"; $s = new Subscriber($feed_hub_url, $callback_url); $rc = $s->subscribe($fetch_url); if ($debug_enabled) { _debug("update_rss_feed: feed hub url found, subscribe request sent."); } db_query($link, "UPDATE ttrss_feeds SET pubsub_state = 1\n\t\t\t\t\t\tWHERE id = '{$feed}'"); } } if ($debug_enabled) { _debug("update_rss_feed: processing articles..."); } foreach ($items as $item) { if ($_REQUEST['xdebug'] == 3) { print_r($item); } $entry_guid = $item->get_id(); if (!$entry_guid) { $entry_guid = $item->get_link(); } if (!$entry_guid) { $entry_guid = make_guid_from_title($item->get_title()); } if ($debug_enabled) { _debug("update_rss_feed: guid {$entry_guid}"); } if (!$entry_guid) { continue; } $entry_guid = "{$owner_uid},{$entry_guid}"; $entry_timestamp = ""; $entry_timestamp = strtotime($item->get_date()); if ($entry_timestamp == -1 || !$entry_timestamp) { $entry_timestamp = time(); $no_orig_date = 'true'; } else { $no_orig_date = 'false'; } $entry_timestamp_fmt = strftime("%Y/%m/%d %H:%M:%S", $entry_timestamp); if ($debug_enabled) { _debug("update_rss_feed: date {$entry_timestamp} [{$entry_timestamp_fmt}]"); } $entry_title = $item->get_title(); $entry_link = rewrite_relative_url($site_url, $item->get_link()); if ($debug_enabled) { _debug("update_rss_feed: title {$entry_title}"); _debug("update_rss_feed: link {$entry_link}"); } if (!$entry_title) { $entry_title = date("Y-m-d H:i:s", $entry_timestamp); } $entry_content = $item->get_content(); if (!$entry_content) { $entry_content = $item->get_description(); } if ($_REQUEST["xdebug"] == 2) { print "update_rss_feed: content: "; print $entry_content; print "\n"; } $entry_comments = $item->data["comments"]; if ($item->get_author()) { $entry_author_item = $item->get_author(); $entry_author = $entry_author_item->get_name(); if (!$entry_author) { $entry_author = $entry_author_item->get_email(); } $entry_author = db_escape_string($entry_author); } $entry_guid = db_escape_string(mb_substr($entry_guid, 0, 245)); $entry_comments = db_escape_string(mb_substr($entry_comments, 0, 245)); $entry_author = db_escape_string(mb_substr($entry_author, 0, 245)); $num_comments = $item->get_item_tags('http://purl.org/rss/1.0/modules/slash/', 'comments'); if (is_array($num_comments) && is_array($num_comments[0])) { $num_comments = (int) $num_comments[0]["data"]; } else { $num_comments = 0; } if ($debug_enabled) { _debug("update_rss_feed: num_comments: {$num_comments}"); _debug("update_rss_feed: looking for tags [1]..."); } // parse <category> entries into tags $additional_tags = array(); $additional_tags_src = $item->get_categories(); if (is_array($additional_tags_src)) { foreach ($additional_tags_src as $tobj) { array_push($additional_tags, $tobj->get_term()); } } if ($debug_enabled) { _debug("update_rss_feed: category tags:"); print_r($additional_tags); } if ($debug_enabled) { _debug("update_rss_feed: looking for tags [2]..."); } $entry_tags = array_unique($additional_tags); for ($i = 0; $i < count($entry_tags); $i++) { $entry_tags[$i] = mb_strtolower($entry_tags[$i], 'utf-8'); } if ($debug_enabled) { //_debug("update_rss_feed: unfiltered tags found:"); //print_r($entry_tags); } if ($debug_enabled) { _debug("update_rss_feed: done collecting data."); } // TODO: less memory-hungry implementation if ($debug_enabled) { _debug("update_rss_feed: applying plugin filters.."); } // FIXME not sure if owner_uid is a good idea here, we may have a base entry without user entry (?) $result = db_query($link, "SELECT plugin_data,title,content,link,tag_cache,author FROM ttrss_entries, ttrss_user_entries\n\t\t\t\t\tWHERE ref_id = id AND guid = '" . db_escape_string($entry_guid) . "' AND owner_uid = {$owner_uid}"); if (db_num_rows($result) != 0) { $entry_plugin_data = db_fetch_result($result, 0, "plugin_data"); $stored_article = array("title" => db_fetch_result($result, 0, "title"), "content" => db_fetch_result($result, 0, "content"), "link" => db_fetch_result($result, 0, "link"), "tags" => explode(",", db_fetch_result($result, 0, "tag_cache")), "author" => db_fetch_result($result, 0, "author")); } else { $entry_plugin_data = ""; $stored_article = array(); } $article = array("owner_uid" => $owner_uid, "guid" => $entry_guid, "title" => $entry_title, "content" => $entry_content, "link" => $entry_link, "tags" => $entry_tags, "plugin_data" => $entry_plugin_data, "author" => $entry_author, "stored" => $stored_article); foreach ($pluginhost->get_hooks($pluginhost::HOOK_ARTICLE_FILTER) as $plugin) { $article = $plugin->hook_article_filter($article); } $entry_tags = $article["tags"]; $entry_guid = db_escape_string($entry_guid); $entry_content = db_escape_string($article["content"], false); $entry_title = db_escape_string($article["title"]); $entry_author = db_escape_string($article["author"]); $entry_link = db_escape_string($article["link"]); $entry_plugin_data = db_escape_string($article["plugin_data"]); if ($debug_enabled) { _debug("update_rss_feed: plugin data: {$entry_plugin_data}"); } if ($cache_images && is_writable(CACHE_DIR . '/images')) { $entry_content = cache_images($entry_content, $site_url, $debug_enabled); } $content_hash = "SHA1:" . sha1($entry_content); db_query($link, "BEGIN"); $result = db_query($link, "SELECT id FROM\tttrss_entries\n\t\t\t\t\tWHERE guid = '{$entry_guid}'"); if (db_num_rows($result) == 0) { if ($debug_enabled) { _debug("update_rss_feed: base guid [{$entry_guid}] not found"); } // base post entry does not exist, create it $result = db_query($link, "INSERT INTO ttrss_entries\n\t\t\t\t\t\t\t(title,\n\t\t\t\t\t\t\tguid,\n\t\t\t\t\t\t\tlink,\n\t\t\t\t\t\t\tupdated,\n\t\t\t\t\t\t\tcontent,\n\t\t\t\t\t\t\tcontent_hash,\n\t\t\t\t\t\t\tcached_content,\n\t\t\t\t\t\t\tno_orig_date,\n\t\t\t\t\t\t\tdate_updated,\n\t\t\t\t\t\t\tdate_entered,\n\t\t\t\t\t\t\tcomments,\n\t\t\t\t\t\t\tnum_comments,\n\t\t\t\t\t\t\tplugin_data,\n\t\t\t\t\t\t\tauthor)\n\t\t\t\t\t\tVALUES\n\t\t\t\t\t\t\t('{$entry_title}',\n\t\t\t\t\t\t\t'{$entry_guid}',\n\t\t\t\t\t\t\t'{$entry_link}',\n\t\t\t\t\t\t\t'{$entry_timestamp_fmt}',\n\t\t\t\t\t\t\t'{$entry_content}',\n\t\t\t\t\t\t\t'{$content_hash}',\n\t\t\t\t\t\t\t'',\n\t\t\t\t\t\t\t{$no_orig_date},\n\t\t\t\t\t\t\tNOW(),\n\t\t\t\t\t\t\tNOW(),\n\t\t\t\t\t\t\t'{$entry_comments}',\n\t\t\t\t\t\t\t'{$num_comments}',\n\t\t\t\t\t\t\t'{$entry_plugin_data}',\n\t\t\t\t\t\t\t'{$entry_author}')"); $article_labels = array(); } else { // we keep encountering the entry in feeds, so we need to // update date_updated column so that we don't get horrible // dupes when the entry gets purged and reinserted again e.g. // in the case of SLOW SLOW OMG SLOW updating feeds $base_entry_id = db_fetch_result($result, 0, "id"); db_query($link, "UPDATE ttrss_entries SET date_updated = NOW()\n\t\t\t\t\t\tWHERE id = '{$base_entry_id}'"); $article_labels = get_article_labels($link, $base_entry_id, $owner_uid); } // now it should exist, if not - bad luck then $result = db_query($link, "SELECT\n\t\t\t\t\t\tid,content_hash,no_orig_date,title,plugin_data,\n\t\t\t\t\t\t" . SUBSTRING_FOR_DATE . "(date_updated,1,19) as date_updated,\n\t\t\t\t\t\t" . SUBSTRING_FOR_DATE . "(updated,1,19) as updated,\n\t\t\t\t\t\tnum_comments\n\t\t\t\t\tFROM\n\t\t\t\t\t\tttrss_entries\n\t\t\t\t\tWHERE guid = '{$entry_guid}'"); $entry_ref_id = 0; $entry_int_id = 0; if (db_num_rows($result) == 1) { if ($debug_enabled) { _debug("update_rss_feed: base guid [{$entry_guid}] found, checking for user record"); } // this will be used below in update handler $orig_content_hash = db_fetch_result($result, 0, "content_hash"); $orig_title = db_fetch_result($result, 0, "title"); $orig_num_comments = db_fetch_result($result, 0, "num_comments"); $orig_date_updated = strtotime(db_fetch_result($result, 0, "date_updated")); $orig_plugin_data = db_fetch_result($result, 0, "plugin_data"); $ref_id = db_fetch_result($result, 0, "id"); $entry_ref_id = $ref_id; // check for user post link to main table // do we allow duplicate posts with same GUID in different feeds? if (get_pref($link, "ALLOW_DUPLICATE_POSTS", $owner_uid, false)) { $dupcheck_qpart = "AND (feed_id = '{$feed}' OR feed_id IS NULL)"; } else { $dupcheck_qpart = ""; } /* Collect article tags here so we could filter by them: */ $article_filters = get_article_filters($filters, $entry_title, $entry_content, $entry_link, $entry_timestamp, $entry_author, $entry_tags); if ($debug_enabled) { _debug("update_rss_feed: article filters: "); if (count($article_filters) != 0) { print_r($article_filters); } } if (find_article_filter($article_filters, "filter")) { db_query($link, "COMMIT"); // close transaction in progress continue; } $score = calculate_article_score($article_filters); if ($debug_enabled) { _debug("update_rss_feed: initial score: {$score}"); } $query = "SELECT ref_id, int_id FROM ttrss_user_entries WHERE\n\t\t\t\t\t\t\tref_id = '{$ref_id}' AND owner_uid = '{$owner_uid}'\n\t\t\t\t\t\t\t{$dupcheck_qpart}"; // if ($_REQUEST["xdebug"]) print "$query\n"; $result = db_query($link, $query); // okay it doesn't exist - create user entry if (db_num_rows($result) == 0) { if ($debug_enabled) { _debug("update_rss_feed: user record not found, creating..."); } if ($score >= -500 && !find_article_filter($article_filters, 'catchup')) { $unread = 'true'; $last_read_qpart = 'NULL'; } else { $unread = 'false'; $last_read_qpart = 'NOW()'; } if (find_article_filter($article_filters, 'mark') || $score > 1000) { $marked = 'true'; } else { $marked = 'false'; } if (find_article_filter($article_filters, 'publish')) { $published = 'true'; } else { $published = 'false'; } // N-grams if (DB_TYPE == "pgsql" and defined('_NGRAM_TITLE_DUPLICATE_THRESHOLD')) { $result = db_query($link, "SELECT COUNT(*) AS similar FROM\n\t\t\t\t\t\t\t\t\tttrss_entries,ttrss_user_entries\n\t\t\t\t\t\t\t\tWHERE ref_id = id AND updated >= NOW() - INTERVAL '7 day'\n\t\t\t\t\t\t\t\t\tAND similarity(title, '{$entry_title}') >= " . _NGRAM_TITLE_DUPLICATE_THRESHOLD . "\n\t\t\t\t\t\t\t\t\tAND owner_uid = {$owner_uid}"); $ngram_similar = db_fetch_result($result, 0, "similar"); if ($debug_enabled) { _debug("update_rss_feed: N-gram similar results: {$ngram_similar}"); } if ($ngram_similar > 0) { $unread = 'false'; } } $result = db_query($link, "INSERT INTO ttrss_user_entries\n\t\t\t\t\t\t\t\t(ref_id, owner_uid, feed_id, unread, last_read, marked,\n\t\t\t\t\t\t\t\t\tpublished, score, tag_cache, label_cache, uuid)\n\t\t\t\t\t\t\tVALUES ('{$ref_id}', '{$owner_uid}', '{$feed}', {$unread},\n\t\t\t\t\t\t\t\t{$last_read_qpart}, {$marked}, {$published}, '{$score}', '', '', '')"); if (PUBSUBHUBBUB_HUB && $published == 'true') { $rss_link = get_self_url_prefix() . "/public.php?op=rss&id=-2&key=" . get_feed_access_key($link, -2, false, $owner_uid); $p = new Publisher(PUBSUBHUBBUB_HUB); $pubsub_result = $p->publish_update($rss_link); } $result = db_query($link, "SELECT int_id FROM ttrss_user_entries WHERE\n\t\t\t\t\t\t\t\tref_id = '{$ref_id}' AND owner_uid = '{$owner_uid}' AND\n\t\t\t\t\t\t\t\tfeed_id = '{$feed}' LIMIT 1"); if (db_num_rows($result) == 1) { $entry_int_id = db_fetch_result($result, 0, "int_id"); } } else { if ($debug_enabled) { _debug("update_rss_feed: user record FOUND"); } $entry_ref_id = db_fetch_result($result, 0, "ref_id"); $entry_int_id = db_fetch_result($result, 0, "int_id"); } if ($debug_enabled) { _debug("update_rss_feed: RID: {$entry_ref_id}, IID: {$entry_int_id}"); } $post_needs_update = false; $update_insignificant = false; if ($orig_num_comments != $num_comments) { $post_needs_update = true; $update_insignificant = true; } if ($entry_plugin_data != $orig_plugin_data) { $post_needs_update = true; $update_insignificant = true; } if ($content_hash != $orig_content_hash) { $post_needs_update = true; $update_insignificant = false; } if (db_escape_string($orig_title) != $entry_title) { $post_needs_update = true; $update_insignificant = false; } // if post needs update, update it and mark all user entries // linking to this post as updated if ($post_needs_update) { if (defined('DAEMON_EXTENDED_DEBUG')) { _debug("update_rss_feed: post {$entry_guid} needs update..."); } // print "<!-- post $orig_title needs update : $post_needs_update -->"; db_query($link, "UPDATE ttrss_entries\n\t\t\t\t\t\t\tSET title = '{$entry_title}', content = '{$entry_content}',\n\t\t\t\t\t\t\t\tcontent_hash = '{$content_hash}',\n\t\t\t\t\t\t\t\tupdated = '{$entry_timestamp_fmt}',\n\t\t\t\t\t\t\t\tnum_comments = '{$num_comments}',\n\t\t\t\t\t\t\t\tplugin_data = '{$entry_plugin_data}'\n\t\t\t\t\t\t\tWHERE id = '{$ref_id}'"); if (!$update_insignificant) { if ($mark_unread_on_update) { db_query($link, "UPDATE ttrss_user_entries\n\t\t\t\t\t\t\t\t\tSET last_read = null, unread = true WHERE ref_id = '{$ref_id}'"); } } } } db_query($link, "COMMIT"); if ($debug_enabled) { _debug("update_rss_feed: assigning labels..."); } assign_article_to_label_filters($link, $entry_ref_id, $article_filters, $owner_uid, $article_labels); if ($debug_enabled) { _debug("update_rss_feed: looking for enclosures..."); } // enclosures $enclosures = array(); $encs = $item->get_enclosures(); if (is_array($encs)) { foreach ($encs as $e) { $e_item = array($e->link, $e->type, $e->length); array_push($enclosures, $e_item); } } if ($debug_enabled) { _debug("update_rss_feed: article enclosures:"); print_r($enclosures); } db_query($link, "BEGIN"); foreach ($enclosures as $enc) { $enc_url = db_escape_string($enc[0]); $enc_type = db_escape_string($enc[1]); $enc_dur = db_escape_string($enc[2]); $result = db_query($link, "SELECT id FROM ttrss_enclosures\n\t\t\t\t\t\tWHERE content_url = '{$enc_url}' AND post_id = '{$entry_ref_id}'"); if (db_num_rows($result) == 0) { db_query($link, "INSERT INTO ttrss_enclosures\n\t\t\t\t\t\t\t(content_url, content_type, title, duration, post_id) VALUES\n\t\t\t\t\t\t\t('{$enc_url}', '{$enc_type}', '', '{$enc_dur}', '{$entry_ref_id}')"); } } db_query($link, "COMMIT"); // check for manual tags (we have to do it here since they're loaded from filters) foreach ($article_filters as $f) { if ($f["type"] == "tag") { $manual_tags = trim_array(explode(",", $f["param"])); foreach ($manual_tags as $tag) { if (tag_is_valid($tag)) { array_push($entry_tags, $tag); } } } } // Skip boring tags $boring_tags = trim_array(explode(",", mb_strtolower(get_pref($link, 'BLACKLISTED_TAGS', $owner_uid, ''), 'utf-8'))); $filtered_tags = array(); $tags_to_cache = array(); if ($entry_tags && is_array($entry_tags)) { foreach ($entry_tags as $tag) { if (array_search($tag, $boring_tags) === false) { array_push($filtered_tags, $tag); } } } $filtered_tags = array_unique($filtered_tags); if ($debug_enabled) { _debug("update_rss_feed: filtered article tags:"); print_r($filtered_tags); } // Save article tags in the database if (count($filtered_tags) > 0) { db_query($link, "BEGIN"); foreach ($filtered_tags as $tag) { $tag = sanitize_tag($tag); $tag = db_escape_string($tag); if (!tag_is_valid($tag)) { continue; } $result = db_query($link, "SELECT id FROM ttrss_tags\n\t\t\t\t\t\t\tWHERE tag_name = '{$tag}' AND post_int_id = '{$entry_int_id}' AND\n\t\t\t\t\t\t\towner_uid = '{$owner_uid}' LIMIT 1"); if ($result && db_num_rows($result) == 0) { db_query($link, "INSERT INTO ttrss_tags\n\t\t\t\t\t\t\t\t\t(owner_uid,tag_name,post_int_id)\n\t\t\t\t\t\t\t\t\tVALUES ('{$owner_uid}','{$tag}', '{$entry_int_id}')"); } array_push($tags_to_cache, $tag); } /* update the cache */ $tags_to_cache = array_unique($tags_to_cache); $tags_str = db_escape_string(join(",", $tags_to_cache)); db_query($link, "UPDATE ttrss_user_entries\n\t\t\t\t\t\tSET tag_cache = '{$tags_str}' WHERE ref_id = '{$entry_ref_id}'\n\t\t\t\t\t\tAND owner_uid = {$owner_uid}"); db_query($link, "COMMIT"); } if (get_pref($link, "AUTO_ASSIGN_LABELS", $owner_uid, false)) { if ($debug_enabled) { _debug("update_rss_feed: auto-assigning labels..."); } foreach ($labels as $label) { $caption = $label["caption"]; if (preg_match("/\\b{$caption}\\b/i", "{$tags_str} " . strip_tags($entry_content) . " {$entry_title}")) { if (!labels_contains_caption($article_labels, $caption)) { label_add_article($link, $entry_ref_id, $caption, $owner_uid); } } } } if ($debug_enabled) { _debug("update_rss_feed: article processed"); } } if (!$last_updated) { if ($debug_enabled) { _debug("update_rss_feed: new feed, catching it up..."); } catchup_feed($link, $feed, false, $owner_uid); } if ($debug_enabled) { _debug("purging feed..."); } purge_feed($link, $feed, 0, $debug_enabled); db_query($link, "UPDATE ttrss_feeds\n\t\t\t\tSET last_updated = NOW(), last_error = '' WHERE id = '{$feed}'"); // db_query($link, "COMMIT"); } else { $error_msg = db_escape_string(mb_substr($rss->error(), 0, 245)); if ($debug_enabled) { _debug("update_rss_feed: error fetching feed: {$error_msg}"); } db_query($link, "UPDATE ttrss_feeds SET last_error = '{$error_msg}',\n\t\t\t\t\tlast_updated = NOW() WHERE id = '{$feed}'"); } unset($rss); if ($debug_enabled) { _debug("update_rss_feed: done"); } }
$feedimport_where .= 'MD5(CONCAT(cnt_id,cnt_text))=' . _dbEscape($feedimport_source); unset($_getVar['feedimport']); $feedimport_result = _dbGet('phpwcms_content', 'cnt_id,cnt_name,cnt_text,cnt_object', $feedimport_where); if (isset($feedimport_result[0]['cnt_id'])) { $feedimport_result = $feedimport_result[0]; $feedimport_result['cnt_object'] = @unserialize($feedimport_result['cnt_object']); } if (isset($feedimport_result['cnt_object']['structure_level_id'])) { // retrieve Feed now // Load SimplePie require_once PHPWCMS_ROOT . '/include/inc_ext/simplepie.inc.php'; $rss_obj = new SimplePie(); // Feed URL $rss_obj->set_feed_url($feedimport_result['cnt_text']); // Output Encoding Charset $rss_obj->set_output_encoding(PHPWCMS_CHARSET); // Disable Feed cache $rss_obj->enable_cache(false); // Remove surrounding DIV $rss_obj->remove_div(true); // Init Feed $rss_obj->init(); if ($rss_obj->data) { $feedimport_result['status'] = array('Feed Importer Status - ' . date('Y-m-d, H:i:s') . LF . '===========================================', $feedimport_result['cnt_name'] . LF . $feedimport_result['cnt_text'] . LF); if (!empty($feedimport_result['cnt_object']['image_url_replace'])) { $feedimport_result['cnt_object']['image_url_replace'] = explode('>', $feedimport_result['cnt_object']['image_url_replace']); $feedimport_result['cnt_object']['image_url_replace'][0] = trim(trim($feedimport_result['cnt_object']['image_url_replace'][0]), '"'); if (isset($feedimport_result['cnt_object']['image_url_replace'][1])) { $feedimport_result['cnt_object']['image_url_replace'][1] = trim(trim($feedimport_result['cnt_object']['image_url_replace'][1]), '"'); } } else {
function generate_content(&$title) { global $serendipity; $number = $this->get_config('number'); $displaydate = $this->get_config('displaydate', 'true'); $dateformat = $this->get_config('dateformat'); $sidebartitle = $title = $this->get_config('sidebartitle', $this->title); $rssuri = $this->get_config('rssuri'); $target = $this->get_config('target'); $cachetime = $this->get_config('cachetime'); $feedtype = $this->get_config('feedtype', 'rss'); $markup = $this->get_config('markup', 'false'); $bulletimg = $this->get_config('bulletimg'); $charset = $this->get_config('charset', 'native'); if (!$number || !is_numeric($number) || $number < 1) { $showAll = true; } else { $showAll = false; } if (!$dateformat || strlen($dateformat) < 1) { $dateformat = '%A, %B %e. %Y'; } if (!$cachetime || !is_numeric($cachetime)) { $cachetime = 10800; // 3 hours in seconds } $smarty = serendipity_db_bool($this->get_config('smarty')); if ($this->get_config('template') != 'plugin_remoterss.tpl') { $smarty = true; } if (trim($rssuri)) { $feedcache = $serendipity['serendipityPath'] . 'templates_c/remoterss_cache_' . md5(preg_replace('@[^a-z0-9]*@i', '', $rssuri) . $this->get_config('template')) . '.dat'; if (!file_exists($feedcache) || filesize($feedcache) == 0 || filemtime($feedcache) < time() - $cachetime) { $this->debug('Cachefile does not existing.'); if (!$this->urlcheck($rssuri)) { $this->debug('URLCheck failed'); echo '<!-- No valid URL! -->'; } elseif ($feedtype == 'rss') { $this->debug('URLCheck succeeded. Touching ' . $feedcache); // Touching the feedcache file will prevent loops of death when the RSS target is the same URI than our blog. @touch($feedcache); require_once S9Y_PEAR_PATH . 'Onyx/RSS.php'; $c = new Onyx_RSS($charset); $this->debug('Running Onyx Parser'); $c->parse($rssuri); $this->encoding = $c->rss['encoding']; $use_rss_link = serendipity_db_bool($this->get_config('use_rss_link')); $rss_elements = explode(',', $this->get_config('show_rss_element')); $escape_rss = serendipity_db_bool($this->get_config('escape_rss')); $i = 0; $content = ''; $smarty_items = array(); while (($showAll || $i < $number) && ($item = $c->getNextItem())) { if (empty($item['title'])) { continue; } $content .= '<div class="rss_item">'; if ($use_rss_link) { $content .= '<div class="rss_link"><a href="' . serendipity_specialchars($this->decode($item['link'])) . '" ' . (!empty($target) ? 'target="' . $target . '"' : '') . '>'; } if (!empty($bulletimg)) { $content .= '<img src="' . $bulletimg . '" border="0" alt="*" /> '; } $is_first = true; foreach ($rss_elements as $rss_element) { $rss_element = trim($rss_element); if (!$is_first) { $content .= '<span class="rss_' . preg_replace('@[^a-z0-9]@imsU', '', $rss_element) . '">'; } if ($escape_rss) { $content .= $this->decode($item[$rss_element]); } else { $content .= serendipity_specialchars($this->decode($item[$rss_element])); } if ($smarty) { $item['display_elements'][preg_replace('@[^a-z0-9]@imsU', '', $rss_element)] = $this->decode($item[$rss_element]); } if (!$is_first) { $content .= '</span>'; } if ($is_first && $use_rss_link) { $content .= '</a></div>'; // end of first linked element } $is_first = false; } if ($is_first && $use_rss_link) { // No XML element has been configured. $content .= '</a></div>'; } $content .= "<br />\n"; $item['timestamp'] = @strtotime(isset($item['pubdate']) ? $item['pubdate'] : $item['dc:date']); if (!($item['timestamp'] == -1) and $displaydate == 'true') { $content .= '<div class="serendipitySideBarDate">' . serendipity_specialchars(serendipity_formatTime($dateformat, $item['timestamp'], false)) . '</div>'; } if ($smarty) { $smarty_items['items'][$i] = $item; $smarty_items['items'][$i]['css_class'] = preg_replace('@[^a-z0-9]@imsU', '', $rss_element); foreach ($item as $key => $val) { $smarty_items['items'][$i]['decoded_' . str_replace(':', '_', $key)] = $this->decode($key); } } $content .= '</div>'; // end of rss_item ++$i; } if ($smarty) { $smarty_items['use_rss_link'] = $use_rss_link; $smarty_items['bulletimg'] = $bulletimg; $smarty_items['escape_rss'] = $escape_rss; $smarty_items['displaydate'] = $displaydate; $smarty_items['dateformat'] = $dateformat; $smarty_items['target'] = $target; $serendipity['smarty']->assign_by_ref('remoterss_items', $smarty_items); $tpl = $this->get_config('template'); if (empty($tpl)) { $tpl = 'plugin_remoterss.tpl'; } // Template specifics go here switch ($tpl) { case 'plugin_remoterss_nasaiotd.tpl': $smarty_items['nasa_image'] = $c->getData('image'); break; } $content = $this->parseTemplate($tpl); } $this->debug('Caching Feed (' . strlen($content) . ' bytes)'); $fp = @fopen($feedcache, 'w'); if (trim($content) != '' && $fp) { fwrite($fp, $content); fclose($fp); $this->debug('Feed cache written'); } else { $this->debug('Could not write (empty?) cache.'); echo '<!-- Cache failed to ' . $feedcache . ' in ' . getcwd() . ' --><br />'; if (trim($content) == '') { $this->debug('Getting old feedcache'); $content = @file_get_contents($feedcache); } } $this->debug('RSS Plugin finished.'); } elseif ($feedtype == 'atom') { $this->debug('URLCheck succeeded. Touching ' . $feedcache); // Touching the feedcache file will prevent loops of death when the RSS target is the same URI than our blog. @touch($feedcache); require_once S9Y_PEAR_PATH . '/simplepie/simplepie.inc'; $this->debug('Running simplepie Parser'); $simplefeed = new SimplePie(); $simplefeed->cache = false; $simplefeed->set_feed_url($rssuri); $success = $simplefeed->init(); $simplefeed->set_output_encoding($charset); $simplefeed->handle_content_type(); $this->encoding = $charset; $use_rss_link = serendipity_db_bool($this->get_config('use_rss_link')); $rss_elements = explode(',', $this->get_config('show_rss_element')); $escape_rss = serendipity_db_bool($this->get_config('escape_rss')); $i = 0; $content = ''; $smarty_items = array(); foreach ($simplefeed->get_items() as $simpleitem) { // map SimplePie items to s9y items $item['title'] = $simpleitem->get_title(); $item['link'] = $simpleitem->get_permalink(); $item['pubdate'] = $simpleitem->get_date('U'); $item['date'] = $simpleitem->get_date('U'); $item['description'] = $simpleitem->get_description(); $item['content'] = $simpleitem->get_content(); $item['author'] = $simpleitem->get_author(); if (!$showAll && $i > $number) { break; } if (empty($item['title'])) { continue; } $content .= '<div class="rss_item">'; if ($use_rss_link) { $content .= '<div class="rss_link"><a href="' . serendipity_specialchars($this->decode($item['link'])) . '" ' . (!empty($target) ? 'target="' . $target . '"' : '') . '>'; } if (!empty($bulletimg)) { $content .= '<img src="' . $bulletimg . '" border="0" alt="*" /> '; } $is_first = true; foreach ($rss_elements as $rss_element) { $rss_element = trim($rss_element); if (!$is_first) { $content .= '<span class="rss_' . preg_replace('@[^a-z0-9]@imsU', '', $rss_element) . '">'; } if ($escape_rss) { $content .= $this->decode($item[$rss_element]); } else { $content .= serendipity_specialchars($this->decode($item[$rss_element])); } if ($smarty) { $item['display_elements'][preg_replace('@[^a-z0-9]@imsU', '', $rss_element)] = $this->decode($item[$rss_element]); } if (!$is_first) { $content .= '</span>'; } if ($is_first && $use_rss_link) { $content .= '</a></div>'; // end of first linked element } $is_first = false; } if ($is_first && $use_rss_link) { // No XML element has been configured. $content .= '</a></div>'; } $content .= "<br />\n"; $item['timestamp'] = @strtotime(isset($item['pubdate']) ? $item['pubdate'] : $item['dc:date']); if (!($item['timestamp'] == -1) and $displaydate == 'true') { $content .= '<div class="serendipitySideBarDate">' . serendipity_specialchars(serendipity_formatTime($dateformat, $item['timestamp'], false)) . '</div>'; } if ($smarty) { $smarty_items['items'][$i] = $item; $smarty_items['items'][$i]['css_class'] = preg_replace('@[^a-z0-9]@imsU', '', $rss_element); foreach ($item as $key => $val) { $smarty_items['items'][$i]['decoded_' . str_replace(':', '_', $key)] = $this->decode($key); } } $content .= '</div>'; // end of rss_item ++$i; } if ($smarty) { $smarty_items['use_rss_link'] = $use_rss_link; $smarty_items['bulletimg'] = $bulletimg; $smarty_items['escape_rss'] = $escape_rss; $smarty_items['displaydate'] = $displaydate; $smarty_items['dateformat'] = $dateformat; $smarty_items['target'] = $target; $serendipity['smarty']->assign_by_ref('remoterss_items', $smarty_items); $tpl = $this->get_config('template'); if (empty($tpl)) { $tpl = 'plugin_remoterss.tpl'; } // Template specifics go here switch ($tpl) { case 'plugin_remoterss_nasaiotd.tpl': $smarty_items['nasa_image'] = $c->getData('image'); break; } $content = $this->parseTemplate($tpl); } $this->debug('Caching Feed (' . strlen($content) . ' bytes)'); $fp = @fopen($feedcache, 'w'); if (trim($content) != '' && $fp) { fwrite($fp, $content); fclose($fp); $this->debug('Feed cache written'); } else { $this->debug('Could not write (empty?) cache.'); echo '<!-- Cache failed to ' . $feedcache . ' in ' . getcwd() . ' --><br />'; if (trim($content) == '') { $this->debug('Getting old feedcache'); $content = @file_get_contents($feedcache); } } $this->debug('RSS Plugin (Atom) finished.'); } elseif ($feedtype == 'opml') { // Touching the feedcache file will prevent loops of death when the RSS target is the same URI than our blog. @touch($feedcache); $opml = new s9y_remoterss_OPML(); $opmltree = $opml->parseOPML($rssuri); if (OPMLDEBUG == 1) { echo "\n<pre>\n"; print_r($opmltree); echo "\n</pre>\n"; } if ($opmltree['tag'] === 'opml') { $head = $opml->getOPMLHead($opmltree); $ownerName = $opml->getOPMLTag($head, 'ownerName'); $blogrolling = $ownerName != false ? $ownerName['value'] == 'Blogroll Owner' ? true : false : false; $i = 0; $content = ''; while (($showAll || $i < $number) && ($item = $opml->getOPMLOutlineAttr($opmltree, $i))) { if (!empty($item['url'])) { $url = $this->decode($item['url']); } elseif (!empty($item['htmlUrl'])) { $url = $this->decode($item['htmlUrl']); } elseif (!empty($item['xmlUrl'])) { $url = $this->decode($item['xmlUrl']); } elseif (!empty($item['urlHTTP'])) { $url = $this->decode($item['urlHTTP']); } else { $url = ''; } if (!empty($item['text'])) { $text = serendipity_specialchars($this->decode($item['text'])); } elseif (!empty($item['title'])) { $text = serendipity_specialchars($this->decode($item['title'])); } elseif (!empty($item['description'])) { $text = serendipity_specialchars($this->decode($item['description'])); } else { $text = ''; } if ($blogrolling === true && (!empty($text) || !empty($url))) { $content .= '• <a href="' . serendipity_specialchars($url) . '" ' . (!empty($target) ? 'target="' . $target . '"' : '') . ' title="' . $text . '">' . $text . "</a>"; if (isset($item['isRecent'])) { $content .= ' <span style="color: Red; ">*</span>'; } $content .= "<br />"; } elseif (isset($item['type']) && $item['type'] == 'url' || !empty($url)) { $content .= '• <a href="' . serendipity_specialchars($url) . '" ' . (!empty($target) ? 'target="' . $target . '"' : '') . ' title="' . $text . '">' . $text . "</a>"; $content .= "<br />"; } ++$i; } /* Pretend to be a html_nugget so we can apply markup events. */ if ($markup == 'true') { $entry = array('html_nugget' => $content); serendipity_plugin_api::hook_event('frontend_display', $entry); $content = $entry['html_nugget']; } $fp = @fopen($feedcache, 'w'); if (trim($content) != '' && $fp) { fwrite($fp, $content); fclose($fp); } else { echo '<!-- Cache failed to ' . $feedcache . ' in ' . getcwd() . ' --><br />'; if (trim($content) == '') { $content = @file_get_contents($feedcache); } } } else { echo '<!-- Not a valid OPML feed -->'; } } else { echo '<!-- no valid feedtype -->'; } } else { $this->debug('Got feed from cache ' . $feedcache); $content = file_get_contents($feedcache); } echo $content; } else { echo PLUGIN_REMOTERSS_NOURI; } }
function plan_read_simplepie_single($url, $id) { $url = str_replace("feed://", "http://", $url); if (!strstr($url, '://')) { $url = plan_get_real_location($url); } $feed = new SimplePie(); $feed->set_feed_url($url); $feed->set_output_encoding('utf-8'); $feed->init(); $feed->handle_content_type(); $these_items = $feed->get_items(); $items = array_merge($items, $these_items); foreach ($items as $item) { if (md5($item->get_id()) == $id) { $item_data = $item->get_content(); //$entry_text=removeEvilTags($item_data); $entry_text = $item_data; $content .= "{$entry_text}"; } } return $content; }
/** * Fetches a feed by URL and caches it - using the SimplePie Library. * * @param string $feed_url This is the URL you want to parse. * @param int $cache_duration This is the number of seconds that you want to store the feedcache file for. * @param string $cache_location This is where you want the cached feeds to be stored. */ public static function fetchRSS($feed_url, $number_of_items = null, $cache_duration = null, $cache_location = null) { /** * SimplePie is a bunch of crap, and not e_strict, yet. hmpf! * Therefore we have to cheat with the error_reporting toggle. * @link: http://tech.groups.yahoo.com/group/simplepie-support/message/3289 */ $old_errorlevel = error_reporting(); error_reporting(0); // load simplepie include ROOT_LIBRARIES . 'simplepie/simplepie.inc'; // instantiate simplepie $simplepie = new \SimplePie(); // if cache_location was not specified manually if ($cache_location == null) { // we set it to the default cache directory for feeds $cache_location = ROOT_CACHE; // . 'feeds'; } // if cache_duration was not specified manually if ($cache_duration == null) { // we set it to the default cache duration time of 1800 $cache_duration = 1800; } // if number of items to fetch is null if ($number_of_items == null) { // we set it to the default value of 5 items $number_of_items = 5; } // finally: fetch the feed and cache it! $simplepie->set_feed_url($feed_url); $simplepie->set_cache_location($cache_location); $simplepie->set_cache_duration($cache_duration); $simplepie->set_timeout(5); $simplepie->set_output_encoding('UTF-8'); $simplepie->set_stupidly_fast(true); $simplepie->init(); $simplepie->handle_content_type(); // set old error reporting level error_reporting($old_errorlevel); return $simplepie; }
public function __construct($data, $boxname = "") { $this->spnrbData['templatename'] = "simplePieNewsreaderBox"; $this->getBoxStatus($data); $this->spnrbData['boxID'] = $data['boxID']; if (SPNRBOX_BOXOPENED == true) { $this->spnrbData['Status'] = 1; } if (WBBCore::getUser()->getPermission('user.board.canViewSimplePieNewsreaderBox')) { require_once WBB_DIR . 'lib/data/boxes/SimplePieNewsReader/simplepie.inc'; require_once WBB_DIR . 'lib/data/boxes/SimplePieNewsReader/idna_convert.class.php'; // FILTER? if (SPNRBOX_FILTER && strlen(SPNRBOX_FILTERWORDS) >= 3) { require_once WBB_DIR . 'lib/data/boxes/SimplePieNewsReader/simplepie_filter.php'; $feed = new SimplePie_Filter(); if (!defined('SPNRBOX_FILTERCLASS')) { define('SPNRBOX_FILTERCLASS', 'hightlight'); } define('SPNRBOX_FILTERON', 1); } else { $feed = new SimplePie(); define('SPNRBOX_FILTERON', 0); } // CACHE if (SPNRBOX_CACHEMAX != 0 && SPNRBOX_CACHEMIN != 0) { $feed->set_autodiscovery_cache_duration(SPNRBOX_CACHEMAX); $feed->set_cache_duration(SPNRBOX_CACHEMIN); } else { $feed->set_autodiscovery_cache_duration(9999999999); $feed->set_cache_duration(9999999999); } // CHARSET if (!defined('CHARSET')) { define('CHARSET', 'UTF-8'); } if (!defined('SPNRBOX_CHARSET')) { define('SPNRBOX_CHARSET', 'UTF-8'); } if (SPNRBOX_CHARSET == 'default') { $charset = CHARSET; } else { $charset = SPNRBOX_CHARSET; } $feed->set_cache_location(WBB_DIR . 'lib/data/boxes/SimplePieNewsReader/cache'); $feed->set_favicon_handler(RELATIVE_WBB_DIR . 'lib/data/boxes/SimplePieNewsReader/handler_image.php'); $feed->set_image_handler(RELATIVE_WBB_DIR . 'lib/data/boxes/SimplePieNewsReader/handler_image.php'); $feed->set_output_encoding($charset); // BOOKMARKS $bookmarks = array(); if (SPNRBOX_SHOWSOCIALBOOKMARKS) { $socialBookmarks = preg_split("/\r?\n/", SPNRBOX_SOCIALBOOKMARKS); $cntBookmark = 0; foreach ($socialBookmarks as $row) { $row = trim($row); if (preg_match("/\\|/", $row)) { list($bookmarkTitle, $bookmarkUrl, $bookmarkImg, $bookmarkEncodeTitle, $bookmarkEncodeUrl) = preg_split("/\\|/", $row, 5); $bookmarkTitle = trim($bookmarkTitle); $bookmarkUrl = trim($bookmarkUrl); $bookmarkImg = trim($bookmarkImg); $bookmarkEncodeTitle = trim($bookmarkEncodeTitle); $bookmarkEncodeUrl = trim($bookmarkEncodeUrl); if (!empty($bookmarkTitle) && !empty($bookmarkUrl) && !empty($bookmarkImg) && isset($bookmarkEncodeTitle) && isset($bookmarkEncodeUrl)) { $bookmarks[$cntBookmark]['bookmarkTitle'] = $bookmarkTitle; $bookmarks[$cntBookmark]['bookmarkUrl'] = $bookmarkUrl; $bookmarks[$cntBookmark]['bookmarkImg'] = $bookmarkImg; $bookmarks[$cntBookmark]['bookmarkEncodeTitle'] = $bookmarkEncodeTitle == 1 ? 1 : 0; $bookmarks[$cntBookmark]['bookmarkEncodeUrl'] = $bookmarkEncodeUrl == 1 ? 1 : 0; $cntBookmark++; } } } } // THEMA ZUM FEED if (WCF::getUser()->getPermission('user.board.canViewThreadToFeed') && SPNRBOX_FEEDTOTHREAD) { require_once WBB_DIR . 'lib/data/board/Board.class.php'; $accessibleBoards = explode(',', Board::getAccessibleBoards()); $selectiveBoards = explode(',', SPNRBOX_FEEDTOTHREADBOARDID); $boardStructur = WCF::getCache()->get('board', 'boardStructure'); if (count($selectiveBoards) != 0) { $this->spnrbData['boardsForm'] = count($selectiveBoards) == 1 ? 'button' : 'list'; $cntBoards = 0; $prefix = ''; foreach ($selectiveBoards as $k => $v) { $tmp = Board::getBoard($v); if ($tmp->boardType < 2 && in_array($v, $accessibleBoards)) { $this->spnrbData['boards'][$cntBoards]['id'] = $tmp->boardID; $this->spnrbData['boards'][$cntBoards]['type'] = $tmp->boardType; $prefix = ''; foreach ($boardStructur as $boardDepth => $boardKey) { if (in_array($this->spnrbData['boards'][$cntBoards]['id'], $boardKey)) { $prefix = str_repeat('--', $boardDepth); break; } } $this->spnrbData['boards'][$cntBoards]['title'] = ($prefix != '' ? $prefix : '') . ' ' . $tmp->title; $cntBoards++; } } } else { $this->spnrbData['boardsForm'] = ''; } } $feedUrls = preg_split('/\\r?\\n/', SPNRBOX_FEEDS); $cntFeedUrl = 0; foreach ($feedUrls as $k => $feedurl) { $feedurl = trim($feedurl); if (empty($feedurl)) { continue; } $feed->set_feed_url($feedurl); $feed->init(); $feed->handle_content_type(); if (SPNRBOX_FILTERON) { $feed->set_filter(SPNRBOX_FILTERWORDS, SPNRBOX_FILTERMODE); } if (!($favicon = $feed->get_favicon())) { $favicon = RELATIVE_WBB_DIR . 'icon/alternate_favicon.png'; } $this->spnrbData['spnrFeeds'][$cntFeedUrl]['id'] = $cntFeedUrl; $this->spnrbData['spnrFeeds'][$cntFeedUrl]['link'] = $feed->get_permalink(); $this->spnrbData['spnrFeeds'][$cntFeedUrl]['title'] = $feed->get_title(); $this->spnrbData['spnrFeeds'][$cntFeedUrl]['favicon'] = $favicon; $this->spnrbData['spnrFeeds'][$cntFeedUrl]['xml'] = $feedurl; $items = $feed->get_items(); if (SPNRBOX_FILTERON) { $items = $feed->filter($items); } $i = 0; foreach ($items as $item) { if ($i >= SPNRBOX_NUMOFFEEDS) { break; } $iFeed = $item->get_feed(); $this->spnrbData['spnrFeeds'][$cntFeedUrl]['iFeed'][$i]['id'] = $i; $this->spnrbData['spnrFeeds'][$cntFeedUrl]['iFeed'][$i]['link'] = $item->get_permalink(); $this->spnrbData['spnrFeeds'][$cntFeedUrl]['iFeed'][$i]['title'] = html_entity_decode($item->get_title(), ENT_QUOTES, $charset); $this->spnrbData['spnrFeeds'][$cntFeedUrl]['iFeed'][$i]['content'] = SPNRBOX_FILTERON ? $this->highlight(SPNRBOX_FILTERWORDS, $item->get_content(), SPNRBOX_FILTERCLASS) : $item->get_content(); $this->spnrbData['spnrFeeds'][$cntFeedUrl]['iFeed'][$i]['date'] = $item->get_date('d.m.Y - H:i:s'); $this->spnrbData['spnrFeeds'][$cntFeedUrl]['iFeed'][$i]['bookmarks'] = array(); if (count($bookmarks)) { $x = 0; foreach ($bookmarks as $bookmark) { $search[0] = "/\\{TITLE\\}/"; $search[1] = "/\\{URL\\}/"; $replace[0] = $bookmark['bookmarkEncodeTitle'] == 1 ? rawurlencode(html_entity_decode($item->get_title(), ENT_QUOTES, $charset)) : html_entity_decode($item->get_title()); $replace[1] = $bookmark['bookmarkEncodeUrl'] == 1 ? rawurlencode(html_entity_decode($item->get_permalink(), ENT_QUOTES, $charset)) : html_entity_decode($item->get_permalink()); $this->spnrbData['spnrFeeds'][$cntFeedUrl]['iFeed'][$i]['bookmarks'][$x]['bookmarkTitle'] = htmlspecialchars($bookmark['bookmarkTitle']); $this->spnrbData['spnrFeeds'][$cntFeedUrl]['iFeed'][$i]['bookmarks'][$x]['bookmarkUrl'] = preg_replace($search, $replace, html_entity_decode($bookmark['bookmarkUrl'])); $this->spnrbData['spnrFeeds'][$cntFeedUrl]['iFeed'][$i]['bookmarks'][$x]['bookmarkImg'] = RELATIVE_WBB_DIR . "icon/" . $bookmark['bookmarkImg']; $x++; } } if ($enclosure = $item->get_enclosure()) { $this->spnrbData['spnrFeeds'][$cntFeedUrl]['iFeed'][$i]['enclosure'] = '<p>' . $enclosure->native_embed(array('audio' => RELATIVE_WBB_DIR . 'icon/place_audio.png', 'video' => RELATIVE_WBB_DIR . 'icon/place_video.png', 'mediaplayer' => RELATIVE_WBB_DIR . 'icon/mediaplayer.swf', 'alt' => '<img src="' . RELATIVE_WBB_DIR . 'icon/mini_podcast.png" class="download" border="0" title="Download Podcast (' . $enclosure->get_extension() . '; ' . $enclosure->get_size() . ' MB)" />', 'altclass' => 'download')) . '</p>'; } $i++; } $cntFeedUrl++; } } }
<?php defined('INTRANET_DIRECTORY') or exit('No direct script access allowed'); $idea = $_SESSION['ide']; $profi = $_SESSION['profi']; // FEED RSS $feed = new SimplePie(); $feed->set_feed_url("http://www.juntadeandalucia.es/educacion/www/novedades.xml"); $feed->set_output_encoding('ISO-8859-1'); $feed->enable_cache(false); $feed->set_cache_duration(600); $feed->init(); $feed->handle_content_type(); $feed->get_title() ? $feed_title = $feed->get_title() : ($feed_title = 'Novedades - Consejería Educación'); $first_items = array(); $items_per_feed = 5; for ($x = 0; $x < $feed->get_item_quantity($items_per_feed); $x++) { $first_items[] = $feed->get_item($x); } ?> <!DOCTYPE html> <html lang="es"> <head> <meta charset="iso-8859-1"> <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1"> <meta name="viewport" content="width=device-width, initial-scale=1.0"> <meta name="description" content="Intranet del <?php echo $config['centro_denominacion']; ?> "> <meta name="author" content="IESMonterroso (https://github.com/IESMonterroso/intranet/)">
public function execute() { parent::execute(); if ($this->action == 'NewsreaderCache') { $urls = preg_split('/\\r?\\n/', SPNRBOX_FEEDS); $cache_location = WBB_DIR . 'lib/data/boxes/SimplePieNewsReader/cache'; // CHARSET if (!defined('CHARSET')) { define('CHARSET', 'UTF-8'); } if (!defined('SPNRBOX_CHARSET')) { define('SPNRBOX_CHARSET', 'UTF-8'); } if (SPNRBOX_CHARSET == 'default') { $charset = CHARSET; } else { $charset = SPNRBOX_CHARSET; } // FILTER? if (SPNRBOX_FILTER && strlen(SPNRBOX_FILTERWORDS) >= 3) { require_once WBB_DIR . 'lib/data/boxes/SimplePieNewsReader/simplepie_filter.php'; $feed = new SimplePie_Filter(); if (!defined('SPNRBOX_FILTERCLASS')) { define('SPNRBOX_FILTERCLASS', 'hightlight'); } define('SPNRBOX_FILTERON', 1); } else { $feed = new SimplePie(); define('SPNRBOX_FILTERON', 0); } $feed->set_feed_url($urls); $feed->set_cache_location($cache_location); $feed->set_autodiscovery_cache_duration(0); $feed->set_cache_duration(0); $feed->set_favicon_handler(RELATIVE_WBB_DIR . 'lib/data/boxes/SimplePieNewsReader/handler_image.php'); $feed->set_image_handler(RELATIVE_WBB_DIR . 'lib/data/boxes/SimplePieNewsReader/handler_image.php'); $feed->set_output_encoding($charset); $feed->set_timeout(10); $feed->init(); $feed->handle_content_type(); if (SPNRBOX_FILTERON) { $feed->set_filter(SPNRBOX_FILTERWORDS, SPNRBOX_FILTERMODE); } header("Content-type: text/plain; charset=UTF-8"); foreach ($urls as $feeds) { $feeds = trim($feeds); if (empty($feeds)) { continue; } $feed->set_feed_url($feeds); $feed->init(); $items = $feed->get_items(); if (SPNRBOX_FILTERON) { $items = $feed->filter($items); } echo $feed->get_title() . "\n"; if (!count($items)) { echo "\tKeine Feeds gefunden.\n"; } else { $i = 0; foreach ($items as $item) { if ($i >= SPNRBOX_NUMOFFEEDS) { break; } SPNRBOX_FILTERON ? $this->highlight(SPNRBOX_FILTERWORDS, $item->get_content(), SPNRBOX_FILTERCLASS) : $item->get_content(); echo "\t\"" . $item->get_title() . "\" -> wurde geladen.\n"; $i++; } } } } }
function check_for_update($link) { $releases_feed = "http://tt-rss.org/releases.rss"; if (!CHECK_FOR_NEW_VERSION || $_SESSION["access_level"] < 10) { return; } error_reporting(0); if (ENABLE_SIMPLEPIE) { $rss = new SimplePie(); $rss->set_useragent(SIMPLEPIE_USERAGENT . MAGPIE_USER_AGENT_EXT); // $rss->set_timeout(MAGPIE_FETCH_TIME_OUT); $rss->set_feed_url($fetch_url); $rss->set_output_encoding('UTF-8'); $rss->init(); } else { $rss = fetch_rss($releases_feed); } error_reporting(DEFAULT_ERROR_LEVEL); if ($rss) { if (ENABLE_SIMPLEPIE) { $items = $rss->get_items(); } else { $items = $rss->items; if (!$items || !is_array($items)) { $items = $rss->entries; } if (!$items || !is_array($items)) { $items = $rss; } } if (!is_array($items) || count($items) == 0) { return; } $latest_item = $items[0]; if (ENABLE_SIMPLEPIE) { $last_title = $latest_item->get_title(); } else { $last_title = $latest_item["title"]; } $latest_version = trim(preg_replace("/(Milestone)|(completed)/", "", $last_title)); if (ENABLE_SIMPLEPIE) { $release_url = sanitize_rss($link, $latest_item->get_link()); $content = sanitize_rss($link, $latest_item->get_description()); } else { $release_url = sanitize_rss($link, $latest_item["link"]); $content = sanitize_rss($link, $latest_item["description"]); } if (version_compare(VERSION, $latest_version) == -1) { return sprintf("New version of Tiny-Tiny RSS (%s) is available:", $latest_version) . "<div class='milestoneDetails'>{$content}</div>"; } else { return false; } } }
} exec("chmod a+rw /tmp/simplepie/."); exec("chmod a+rw /tmp/simplepie/cache/."); require_once "simplepie/simplepie.inc"; function textLimit($string, $length, $replacer = '...') { if (strlen($string) > $length) { return (preg_match('/^(.*)\\W.*$/', substr($string, 0, $length + 1), $matches) ? $matches[1] : substr($string, 0, $length)) . $replacer; } return $string; } $feed = new SimplePie(); $feed->set_cache_location("/tmp/simplepie/"); $feed->set_feed_url($rss_feed_s); $feed->init(); $feed->set_output_encoding('latin-1'); $feed->handle_content_type(); $counter = 1; foreach ($feed->get_items() as $item) { $feed = $item->get_feed(); $feed->strip_htmltags(); echo "<a target='blank' href='" . $item->get_permalink() . "'>" . $item->get_title() . "</a><br />"; $content = $item->get_content(); $content = strip_tags($content); echo textLimit($content, $rsswidgettextlength) . "<br />"; echo "Source: <a target='_blank' href='" . $item->get_permalink() . "'><img src='" . $feed->get_favicon() . "' alt='" . $feed->get_title() . "' title='" . $feed->get_title() . "' border='0' width='16' height='16' /></a><br />"; $counter++; if ($counter > $max_items) { break; } echo "<hr/>";
/** * Build SimplePie object based on RSS or Atom feed from URL. * * @since 2.8.0 * * @param mixed $url URL of feed to retrieve. If an array of URLs, the feeds are merged * using SimplePie's multifeed feature. * See also {@link http://simplepie.org/wiki/faq/typical_multifeed_gotchas} * * @return WP_Error|SimplePie WP_Error object on failure or SimplePie object on success */ function fetch_feed($url) { require_once ABSPATH . WPINC . '/class-feed.php'; $feed = new SimplePie(); $feed->set_sanitize_class('WP_SimplePie_Sanitize_KSES'); // We must manually overwrite $feed->sanitize because SimplePie's // constructor sets it before we have a chance to set the sanitization class $feed->sanitize = new WP_SimplePie_Sanitize_KSES(); $feed->set_cache_class('WP_Feed_Cache'); $feed->set_file_class('WP_SimplePie_File'); $feed->set_feed_url($url); /** This filter is documented in wp-includes/class-feed.php */ $feed->set_cache_duration(apply_filters('wp_feed_cache_transient_lifetime', 12 * HOUR_IN_SECONDS, $url)); /** * Fires just before processing the SimplePie feed object. * * @since 3.0.0 * * @param object &$feed SimplePie feed object, passed by reference. * @param mixed $url URL of feed to retrieve. If an array of URLs, the feeds are merged. */ do_action_ref_array('wp_feed_options', array(&$feed, $url)); $feed->init(); $feed->set_output_encoding(get_option('blog_charset')); $feed->handle_content_type(); if ($feed->error()) { return new WP_Error('simplepie-error', $feed->error()); } return $feed; }
$feed = new SimplePie(); // Setting mutiple RSS feeds to parse: $feed->set_feed_url(array('http://feeds.feedburner.com/MumbrellaAsia', 'http://feeds.feedburner.com/campaignasia', 'http://www.marketing-interactive.com/feed', 'http://rightintel.com/posts/rss/a3835188ea9b3034b43448c220973106d71b2aa5/intel.rss')); // If want single URL: //$feed->set_feed_url('https://rightintel.com/posts/rss/a3835188ea9b3034b43448c220973106d71b2aa5/intel.rss'); // Naming sources here // NOTE that this is for matching with permalink url. Every feed has a unique feed URL, so double check first $mumbrella = 'r/MumbrellaAsia/'; $campaignasia = 'campaignasia.feedsportal.com'; $marketingInteractive = 'marketing-interactive.com'; $rightIntel = '<img src="https://rightintel.com/'; // Set a folder where the cache can reside. // $feed->set_cache_location($_SERVER['DOCUMENT_ROOT'] . '/CodeIgniter/application/cache'); $feed->set_cache_location('/home/weiwenng/public_html/tbwa/cache'); // Ensure the output from the RSS feed matches the MySQL encoding, otherwise you end up with characters like this: †$feed->set_output_encoding('UTF-8'); // Initialize SimplePie. $feed->init(); // This makes sure that the content is sent to the browser as text/html and the UTF-8 character set (since we didn't change it). $feed->handle_content_type(); // Loop through the parsed items and load them to the database. foreach ($feed->get_items() as $item) { // fresh out the source for matching $source = ''; // Items pulled from RSS feed: $title = $item->get_title(); $content = $item->get_description(); $date = $item->get_date('Y-m-j g:i:s'); //format date here $permalink = $item->get_permalink(); // Adding source
function update_rss_feed($link, $feed, $ignore_daemon = false, $no_cache = false, $override_url = false) { require_once "lib/simplepie/simplepie.inc"; require_once "lib/magpierss/rss_fetch.inc"; require_once 'lib/magpierss/rss_utils.inc'; $debug_enabled = defined('DAEMON_EXTENDED_DEBUG') || $_REQUEST['xdebug']; if (!$_REQUEST["daemon"] && !$ignore_daemon) { return false; } if ($debug_enabled) { _debug("update_rss_feed: start"); } if (!$ignore_daemon) { if (DB_TYPE == "pgsql") { $updstart_thresh_qpart = "(ttrss_feeds.last_update_started IS NULL OR ttrss_feeds.last_update_started < NOW() - INTERVAL '120 seconds')"; } else { $updstart_thresh_qpart = "(ttrss_feeds.last_update_started IS NULL OR ttrss_feeds.last_update_started < DATE_SUB(NOW(), INTERVAL 120 SECOND))"; } $result = db_query($link, "SELECT id,update_interval,auth_login,\n\t\t\t\tauth_pass,cache_images,update_method,last_updated\n\t\t\t\tFROM ttrss_feeds WHERE id = '{$feed}' AND {$updstart_thresh_qpart}"); } else { $result = db_query($link, "SELECT id,update_interval,auth_login,\n\t\t\t\tfeed_url,auth_pass,cache_images,update_method,last_updated,\n\t\t\t\tmark_unread_on_update, owner_uid, update_on_checksum_change,\n\t\t\t\tpubsub_state\n\t\t\t\tFROM ttrss_feeds WHERE id = '{$feed}'"); } if (db_num_rows($result) == 0) { if ($debug_enabled) { _debug("update_rss_feed: feed {$feed} NOT FOUND/SKIPPED"); } return false; } $update_method = db_fetch_result($result, 0, "update_method"); $last_updated = db_fetch_result($result, 0, "last_updated"); $owner_uid = db_fetch_result($result, 0, "owner_uid"); $mark_unread_on_update = sql_bool_to_bool(db_fetch_result($result, 0, "mark_unread_on_update")); $update_on_checksum_change = sql_bool_to_bool(db_fetch_result($result, 0, "update_on_checksum_change")); $pubsub_state = db_fetch_result($result, 0, "pubsub_state"); db_query($link, "UPDATE ttrss_feeds SET last_update_started = NOW()\n\t\t\tWHERE id = '{$feed}'"); $auth_login = db_fetch_result($result, 0, "auth_login"); $auth_pass = db_fetch_result($result, 0, "auth_pass"); if ($update_method == 0) { $update_method = DEFAULT_UPDATE_METHOD + 1; } // 1 - Magpie // 2 - SimplePie // 3 - Twitter OAuth if ($update_method == 2) { $use_simplepie = true; } else { $use_simplepie = false; } if ($debug_enabled) { _debug("update method: {$update_method} (feed setting: {$update_method}) (use simplepie: {$use_simplepie})\n"); } if ($update_method == 1) { $auth_login = urlencode($auth_login); $auth_pass = urlencode($auth_pass); } $cache_images = sql_bool_to_bool(db_fetch_result($result, 0, "cache_images")); $fetch_url = db_fetch_result($result, 0, "feed_url"); $feed = db_escape_string($feed); if ($auth_login && $auth_pass) { $url_parts = array(); preg_match("/(^[^:]*):\\/\\/(.*)/", $fetch_url, $url_parts); if ($url_parts[1] && $url_parts[2]) { $fetch_url = $url_parts[1] . "://{$auth_login}:{$auth_pass}@" . $url_parts[2]; } } if ($override_url) { $fetch_url = $override_url; } if ($debug_enabled) { _debug("update_rss_feed: fetching [{$fetch_url}]..."); } // Ignore cache if new feed or manual update. $cache_age = is_null($last_updated) || $last_updated == '1970-01-01 00:00:00' ? -1 : get_feed_update_interval($link, $feed) * 60; if ($update_method == 3) { $rss = fetch_twitter_rss($link, $fetch_url, $owner_uid); } else { if ($update_method == 1) { define('MAGPIE_CACHE_AGE', $cache_age); define('MAGPIE_CACHE_ON', !$no_cache); define('MAGPIE_FETCH_TIME_OUT', 60); define('MAGPIE_CACHE_DIR', CACHE_DIR . "/magpie"); $rss = @fetch_rss($fetch_url); } else { $simplepie_cache_dir = CACHE_DIR . "/simplepie"; if (!is_dir($simplepie_cache_dir)) { mkdir($simplepie_cache_dir); } $rss = new SimplePie(); $rss->set_useragent(SELF_USER_AGENT); # $rss->set_timeout(10); $rss->set_feed_url($fetch_url); $rss->set_output_encoding('UTF-8'); //$rss->force_feed(true); if ($debug_enabled) { _debug("feed update interval (sec): " . get_feed_update_interval($link, $feed) * 60); } $rss->enable_cache(!$no_cache); if (!$no_cache) { $rss->set_cache_location($simplepie_cache_dir); $rss->set_cache_duration($cache_age); } $rss->init(); } } // print_r($rss); if ($debug_enabled) { _debug("update_rss_feed: fetch done, parsing..."); } $feed = db_escape_string($feed); if ($update_method == 2) { $fetch_ok = !$rss->error(); } else { $fetch_ok = !!$rss; } if ($fetch_ok) { if ($debug_enabled) { _debug("update_rss_feed: processing feed data..."); } // db_query($link, "BEGIN"); if (DB_TYPE == "pgsql") { $favicon_interval_qpart = "favicon_last_checked < NOW() - INTERVAL '12 hour'"; } else { $favicon_interval_qpart = "favicon_last_checked < DATE_SUB(NOW(), INTERVAL 12 HOUR)"; } $result = db_query($link, "SELECT title,icon_url,site_url,owner_uid,\n\t\t\t\t(favicon_last_checked IS NULL OR {$favicon_interval_qpart}) AS\n\t\t\t\t\t\tfavicon_needs_check\n\t\t\t\tFROM ttrss_feeds WHERE id = '{$feed}'"); $registered_title = db_fetch_result($result, 0, "title"); $orig_icon_url = db_fetch_result($result, 0, "icon_url"); $orig_site_url = db_fetch_result($result, 0, "site_url"); $favicon_needs_check = sql_bool_to_bool(db_fetch_result($result, 0, "favicon_needs_check")); $owner_uid = db_fetch_result($result, 0, "owner_uid"); if ($use_simplepie) { $site_url = db_escape_string(trim($rss->get_link())); } else { $site_url = db_escape_string(trim($rss->channel["link"])); } // weird, weird Magpie if (!$use_simplepie) { if (!$site_url) { $site_url = db_escape_string($rss->channel["link_"]); } } $site_url = rewrite_relative_url($fetch_url, $site_url); $site_url = substr($site_url, 0, 250); if ($debug_enabled) { _debug("update_rss_feed: checking favicon..."); } if ($favicon_needs_check) { check_feed_favicon($site_url, $feed, $link); db_query($link, "UPDATE ttrss_feeds SET favicon_last_checked = NOW()\n\t\t\t\t\tWHERE id = '{$feed}'"); } if (!$registered_title || $registered_title == "[Unknown]") { if ($use_simplepie) { $feed_title = db_escape_string($rss->get_title()); } else { $feed_title = db_escape_string($rss->channel["title"]); } if ($debug_enabled) { _debug("update_rss_feed: registering title: {$feed_title}"); } db_query($link, "UPDATE ttrss_feeds SET\n\t\t\t\t\ttitle = '{$feed_title}' WHERE id = '{$feed}'"); } if ($site_url && $orig_site_url != $site_url) { db_query($link, "UPDATE ttrss_feeds SET\n\t\t\t\t\tsite_url = '{$site_url}' WHERE id = '{$feed}'"); } // print "I: " . $rss->channel["image"]["url"]; if (!$use_simplepie) { $icon_url = db_escape_string(trim($rss->image["url"])); } else { $icon_url = db_escape_string(trim($rss->get_image_url())); } $icon_url = rewrite_relative_url($fetch_url, $icon_url); $icon_url = substr($icon_url, 0, 250); if ($icon_url && $orig_icon_url != $icon_url) { db_query($link, "UPDATE ttrss_feeds SET icon_url = '{$icon_url}' WHERE id = '{$feed}'"); } if ($debug_enabled) { _debug("update_rss_feed: loading filters..."); } $filters = load_filters($link, $feed, $owner_uid); // if ($debug_enabled) { // print_r($filters); // } if ($use_simplepie) { $iterator = $rss->get_items(); } else { $iterator = $rss->items; if (!$iterator || !is_array($iterator)) { $iterator = $rss->entries; } if (!$iterator || !is_array($iterator)) { $iterator = $rss; } } if (!is_array($iterator)) { /* db_query($link, "UPDATE ttrss_feeds SET last_error = 'Parse error: can\'t find any articles.' WHERE id = '$feed'"); */ // clear any errors and mark feed as updated if fetched okay // even if it's blank if ($debug_enabled) { _debug("update_rss_feed: entry iterator is not an array, no articles?"); } db_query($link, "UPDATE ttrss_feeds\n\t\t\t\t\tSET last_updated = NOW(), last_error = '' WHERE id = '{$feed}'"); return; // no articles } if ($pubsub_state != 2 && PUBSUBHUBBUB_ENABLED) { if ($debug_enabled) { _debug("update_rss_feed: checking for PUSH hub..."); } $feed_hub_url = false; if ($use_simplepie) { $links = $rss->get_links('hub'); if ($links && is_array($links)) { foreach ($links as $l) { $feed_hub_url = $l; break; } } } else { $atom = $rss->channel['atom']; if ($atom) { if ($atom['link@rel'] == 'hub') { $feed_hub_url = $atom['link@href']; } if (!$feed_hub_url && $atom['link#'] > 1) { for ($i = 2; $i <= $atom['link#']; $i++) { if ($atom["link#{$i}@rel"] == 'hub') { $feed_hub_url = $atom["link#{$i}@href"]; break; } } } } else { $feed_hub_url = $rss->channel['link_hub']; } } if ($debug_enabled) { _debug("update_rss_feed: feed hub url: {$feed_hub_url}"); } if ($feed_hub_url && function_exists('curl_init') && !ini_get("open_basedir")) { require_once 'lib/pubsubhubbub/subscriber.php'; $callback_url = get_self_url_prefix() . "/public.php?op=pubsub&id={$feed}"; $s = new Subscriber($feed_hub_url, $callback_url); $rc = $s->subscribe($fetch_url); if ($debug_enabled) { _debug("update_rss_feed: feed hub url found, subscribe request sent."); } db_query($link, "UPDATE ttrss_feeds SET pubsub_state = 1\n\t\t\t\t\t\tWHERE id = '{$feed}'"); } } if ($debug_enabled) { _debug("update_rss_feed: processing articles..."); } foreach ($iterator as $item) { if ($_REQUEST['xdebug'] == 2) { print_r($item); } if ($use_simplepie) { $entry_guid = $item->get_id(); if (!$entry_guid) { $entry_guid = $item->get_link(); } if (!$entry_guid) { $entry_guid = make_guid_from_title($item->get_title()); } } else { $entry_guid = $item["id"]; if (!$entry_guid) { $entry_guid = $item["guid"]; } if (!$entry_guid) { $entry_guid = $item["about"]; } if (!$entry_guid) { $entry_guid = $item["link"]; } if (!$entry_guid) { $entry_guid = make_guid_from_title($item["title"]); } } if ($debug_enabled) { _debug("update_rss_feed: guid {$entry_guid}"); } if (!$entry_guid) { continue; } $entry_timestamp = ""; if ($use_simplepie) { $entry_timestamp = strtotime($item->get_date()); } else { $rss_2_date = $item['pubdate']; $rss_1_date = $item['dc']['date']; $atom_date = $item['issued']; if (!$atom_date) { $atom_date = $item['updated']; } if ($atom_date != "") { $entry_timestamp = parse_w3cdtf($atom_date); } if ($rss_1_date != "") { $entry_timestamp = parse_w3cdtf($rss_1_date); } if ($rss_2_date != "") { $entry_timestamp = strtotime($rss_2_date); } } if ($entry_timestamp == "" || $entry_timestamp == -1 || !$entry_timestamp) { $entry_timestamp = time(); $no_orig_date = 'true'; } else { $no_orig_date = 'false'; } $entry_timestamp_fmt = strftime("%Y/%m/%d %H:%M:%S", $entry_timestamp); if ($debug_enabled) { _debug("update_rss_feed: date {$entry_timestamp} [{$entry_timestamp_fmt}]"); } if ($use_simplepie) { $entry_title = $item->get_title(); } else { $entry_title = trim(strip_tags($item["title"])); } if ($use_simplepie) { $entry_link = $item->get_link(); } else { // strange Magpie workaround $entry_link = $item["link_"]; if (!$entry_link) { $entry_link = $item["link"]; } } $entry_link = rewrite_relative_url($site_url, $entry_link); if ($debug_enabled) { _debug("update_rss_feed: title {$entry_title}"); _debug("update_rss_feed: link {$entry_link}"); } if (!$entry_title) { $entry_title = date("Y-m-d H:i:s", $entry_timestamp); } $entry_link = strip_tags($entry_link); if ($use_simplepie) { $entry_content = $item->get_content(); if (!$entry_content) { $entry_content = $item->get_description(); } } else { $entry_content = $item["content:escaped"]; if (!$entry_content) { $entry_content = $item["content:encoded"]; } if (!$entry_content && is_array($entry_content)) { $entry_content = $item["content"]["encoded"]; } if (!$entry_content) { $entry_content = $item["content"]; } if (is_array($entry_content)) { $entry_content = $entry_content[0]; } // Magpie bugs are getting ridiculous if (trim($entry_content) == "Array") { $entry_content = false; } if (!$entry_content) { $entry_content = $item["atom_content"]; } if (!$entry_content) { $entry_content = $item["summary"]; } if (!$entry_content || strlen($entry_content) < strlen($item["description"])) { $entry_content = $item["description"]; } // WTF if (is_array($entry_content)) { $entry_content = $entry_content["encoded"]; if (!$entry_content) { $entry_content = $entry_content["escaped"]; } } } if ($cache_images && is_writable(CACHE_DIR . '/images')) { $entry_content = cache_images($entry_content, $site_url, $debug_enabled); } if ($_REQUEST["xdebug"] == 2) { print "update_rss_feed: content: "; print $entry_content; print "\n"; } $entry_content_unescaped = $entry_content; if ($use_simplepie) { $entry_comments = strip_tags($item->data["comments"]); if ($item->get_author()) { $entry_author_item = $item->get_author(); $entry_author = $entry_author_item->get_name(); if (!$entry_author) { $entry_author = $entry_author_item->get_email(); } $entry_author = db_escape_string($entry_author); } } else { $entry_comments = strip_tags($item["comments"]); $entry_author = db_escape_string(strip_tags($item['dc']['creator'])); if ($item['author']) { if (is_array($item['author'])) { if (!$entry_author) { $entry_author = db_escape_string(strip_tags($item['author']['name'])); } if (!$entry_author) { $entry_author = db_escape_string(strip_tags($item['author']['email'])); } } if (!$entry_author) { $entry_author = db_escape_string(strip_tags($item['author'])); } } } if (preg_match('/^[\\t\\n\\r ]*$/', $entry_author)) { $entry_author = ''; } $entry_guid = db_escape_string(strip_tags($entry_guid)); $entry_guid = mb_substr($entry_guid, 0, 250); $result = db_query($link, "SELECT id FROM\tttrss_entries\n\t\t\t\t\tWHERE guid = '{$entry_guid}'"); $entry_content = db_escape_string($entry_content, false); $content_hash = "SHA1:" . sha1(strip_tags($entry_content)); $entry_title = db_escape_string($entry_title); $entry_link = db_escape_string($entry_link); $entry_comments = mb_substr(db_escape_string($entry_comments), 0, 250); $entry_author = mb_substr($entry_author, 0, 250); if ($use_simplepie) { $num_comments = 0; #FIXME# } else { $num_comments = db_escape_string($item["slash"]["comments"]); } if (!$num_comments) { $num_comments = 0; } if ($debug_enabled) { _debug("update_rss_feed: looking for tags [1]..."); } // parse <category> entries into tags $additional_tags = array(); if ($use_simplepie) { $additional_tags_src = $item->get_categories(); if (is_array($additional_tags_src)) { foreach ($additional_tags_src as $tobj) { array_push($additional_tags, $tobj->get_term()); } } if ($debug_enabled) { _debug("update_rss_feed: category tags:"); print_r($additional_tags); } } else { $t_ctr = $item['category#']; if ($t_ctr == 0) { $additional_tags = array(); } else { if ($t_ctr > 0) { $additional_tags = array($item['category']); if ($item['category@term']) { array_push($additional_tags, $item['category@term']); } for ($i = 0; $i <= $t_ctr; $i++) { if ($item["category#{$i}"]) { array_push($additional_tags, $item["category#{$i}"]); } if ($item["category#{$i}@term"]) { array_push($additional_tags, $item["category#{$i}@term"]); } } } } // parse <dc:subject> elements $t_ctr = $item['dc']['subject#']; if ($t_ctr > 0) { array_push($additional_tags, $item['dc']['subject']); for ($i = 0; $i <= $t_ctr; $i++) { if ($item['dc']["subject#{$i}"]) { array_push($additional_tags, $item['dc']["subject#{$i}"]); } } } } if ($debug_enabled) { _debug("update_rss_feed: looking for tags [2]..."); } /* taaaags */ // <a href="..." rel="tag">Xorg</a>, // $entry_tags = null; preg_match_all("/<a.*?rel=['\"]tag['\"].*?\\>([^<]+)<\\/a>/i", $entry_content_unescaped, $entry_tags); $entry_tags = $entry_tags[1]; $entry_tags = array_merge($entry_tags, $additional_tags); $entry_tags = array_unique($entry_tags); for ($i = 0; $i < count($entry_tags); $i++) { $entry_tags[$i] = mb_strtolower($entry_tags[$i], 'utf-8'); } if ($debug_enabled) { //_debug("update_rss_feed: unfiltered tags found:"); //print_r($entry_tags); } # sanitize content $entry_content = sanitize_article_content($entry_content); $entry_title = sanitize_article_content($entry_title); if ($debug_enabled) { _debug("update_rss_feed: done collecting data [TITLE:{$entry_title}]"); } db_query($link, "BEGIN"); if (db_num_rows($result) == 0) { if ($debug_enabled) { _debug("update_rss_feed: base guid not found"); } // base post entry does not exist, create it $result = db_query($link, "INSERT INTO ttrss_entries\n\t\t\t\t\t\t\t(title,\n\t\t\t\t\t\t\tguid,\n\t\t\t\t\t\t\tlink,\n\t\t\t\t\t\t\tupdated,\n\t\t\t\t\t\t\tcontent,\n\t\t\t\t\t\t\tcontent_hash,\n\t\t\t\t\t\t\tno_orig_date,\n\t\t\t\t\t\t\tdate_updated,\n\t\t\t\t\t\t\tdate_entered,\n\t\t\t\t\t\t\tcomments,\n\t\t\t\t\t\t\tnum_comments,\n\t\t\t\t\t\t\tauthor)\n\t\t\t\t\t\tVALUES\n\t\t\t\t\t\t\t('{$entry_title}',\n\t\t\t\t\t\t\t'{$entry_guid}',\n\t\t\t\t\t\t\t'{$entry_link}',\n\t\t\t\t\t\t\t'{$entry_timestamp_fmt}',\n\t\t\t\t\t\t\t'{$entry_content}',\n\t\t\t\t\t\t\t'{$content_hash}',\n\t\t\t\t\t\t\t{$no_orig_date},\n\t\t\t\t\t\t\tNOW(),\n\t\t\t\t\t\t\tNOW(),\n\t\t\t\t\t\t\t'{$entry_comments}',\n\t\t\t\t\t\t\t'{$num_comments}',\n\t\t\t\t\t\t\t'{$entry_author}')"); } else { // we keep encountering the entry in feeds, so we need to // update date_updated column so that we don't get horrible // dupes when the entry gets purged and reinserted again e.g. // in the case of SLOW SLOW OMG SLOW updating feeds $base_entry_id = db_fetch_result($result, 0, "id"); db_query($link, "UPDATE ttrss_entries SET date_updated = NOW()\n\t\t\t\t\t\tWHERE id = '{$base_entry_id}'"); } // now it should exist, if not - bad luck then $result = db_query($link, "SELECT\n\t\t\t\t\t\tid,content_hash,no_orig_date,title,\n\t\t\t\t\t\t" . SUBSTRING_FOR_DATE . "(date_updated,1,19) as date_updated,\n\t\t\t\t\t\t" . SUBSTRING_FOR_DATE . "(updated,1,19) as updated,\n\t\t\t\t\t\tnum_comments\n\t\t\t\t\tFROM\n\t\t\t\t\t\tttrss_entries\n\t\t\t\t\tWHERE guid = '{$entry_guid}'"); $entry_ref_id = 0; $entry_int_id = 0; if (db_num_rows($result) == 1) { if ($debug_enabled) { _debug("update_rss_feed: base guid found, checking for user record"); } // this will be used below in update handler $orig_content_hash = db_fetch_result($result, 0, "content_hash"); $orig_title = db_fetch_result($result, 0, "title"); $orig_num_comments = db_fetch_result($result, 0, "num_comments"); $orig_date_updated = strtotime(db_fetch_result($result, 0, "date_updated")); $ref_id = db_fetch_result($result, 0, "id"); $entry_ref_id = $ref_id; // check for user post link to main table // do we allow duplicate posts with same GUID in different feeds? if (get_pref($link, "ALLOW_DUPLICATE_POSTS", $owner_uid, false)) { $dupcheck_qpart = "AND (feed_id = '{$feed}' OR feed_id IS NULL)"; } else { $dupcheck_qpart = ""; } /* Collect article tags here so we could filter by them: */ $article_filters = get_article_filters($filters, $entry_title, $entry_content, $entry_link, $entry_timestamp, $entry_author, $entry_tags); if ($debug_enabled) { _debug("update_rss_feed: article filters: "); if (count($article_filters) != 0) { print_r($article_filters); } } if (find_article_filter($article_filters, "filter")) { db_query($link, "COMMIT"); // close transaction in progress continue; } $score = calculate_article_score($article_filters); if ($debug_enabled) { _debug("update_rss_feed: initial score: {$score}"); } $query = "SELECT ref_id, int_id FROM ttrss_user_entries WHERE\n\t\t\t\t\t\t\tref_id = '{$ref_id}' AND owner_uid = '{$owner_uid}'\n\t\t\t\t\t\t\t{$dupcheck_qpart}"; // if ($_REQUEST["xdebug"]) print "$query\n"; $result = db_query($link, $query); // okay it doesn't exist - create user entry if (db_num_rows($result) == 0) { if ($debug_enabled) { _debug("update_rss_feed: user record not found, creating..."); } if ($score >= -500 && !find_article_filter($article_filters, 'catchup')) { $unread = 'true'; $last_read_qpart = 'NULL'; } else { $unread = 'false'; $last_read_qpart = 'NOW()'; } if (find_article_filter($article_filters, 'mark') || $score > 1000) { $marked = 'true'; } else { $marked = 'false'; } if (find_article_filter($article_filters, 'publish')) { $published = 'true'; } else { $published = 'false'; } // N-grams if (DB_TYPE == "pgsql" and defined('_NGRAM_TITLE_DUPLICATE_THRESHOLD')) { $result = db_query($link, "SELECT COUNT(*) AS similar FROM\n\t\t\t\t\t\t\t\t\tttrss_entries,ttrss_user_entries\n\t\t\t\t\t\t\t\tWHERE ref_id = id AND updated >= NOW() - INTERVAL '7 day'\n\t\t\t\t\t\t\t\t\tAND similarity(title, '{$entry_title}') >= " . _NGRAM_TITLE_DUPLICATE_THRESHOLD . "\n\t\t\t\t\t\t\t\t\tAND owner_uid = {$owner_uid}"); $ngram_similar = db_fetch_result($result, 0, "similar"); if ($debug_enabled) { _debug("update_rss_feed: N-gram similar results: {$ngram_similar}"); } if ($ngram_similar > 0) { $unread = 'false'; } } $result = db_query($link, "INSERT INTO ttrss_user_entries\n\t\t\t\t\t\t\t\t(ref_id, owner_uid, feed_id, unread, last_read, marked,\n\t\t\t\t\t\t\t\t\tpublished, score, tag_cache, label_cache, uuid)\n\t\t\t\t\t\t\tVALUES ('{$ref_id}', '{$owner_uid}', '{$feed}', {$unread},\n\t\t\t\t\t\t\t\t{$last_read_qpart}, {$marked}, {$published}, '{$score}', '', '', '')"); if (PUBSUBHUBBUB_HUB && $published == 'true') { $rss_link = get_self_url_prefix() . "/public.php?op=rss&id=-2&key=" . get_feed_access_key($link, -2, false, $owner_uid); $p = new Publisher(PUBSUBHUBBUB_HUB); $pubsub_result = $p->publish_update($rss_link); } $result = db_query($link, "SELECT int_id FROM ttrss_user_entries WHERE\n\t\t\t\t\t\t\t\tref_id = '{$ref_id}' AND owner_uid = '{$owner_uid}' AND\n\t\t\t\t\t\t\t\tfeed_id = '{$feed}' LIMIT 1"); if (db_num_rows($result) == 1) { $entry_int_id = db_fetch_result($result, 0, "int_id"); } } else { if ($debug_enabled) { _debug("update_rss_feed: user record FOUND"); } $entry_ref_id = db_fetch_result($result, 0, "ref_id"); $entry_int_id = db_fetch_result($result, 0, "int_id"); } if ($debug_enabled) { _debug("update_rss_feed: RID: {$entry_ref_id}, IID: {$entry_int_id}"); } $post_needs_update = false; $update_insignificant = false; if ($orig_num_comments != $num_comments) { $post_needs_update = true; $update_insignificant = true; } if ($content_hash != $orig_content_hash) { $post_needs_update = true; $update_insignificant = false; } if (db_escape_string($orig_title) != $entry_title) { $post_needs_update = true; $update_insignificant = false; } // if post needs update, update it and mark all user entries // linking to this post as updated if ($post_needs_update) { if (defined('DAEMON_EXTENDED_DEBUG')) { _debug("update_rss_feed: post {$entry_guid} needs update..."); } // print "<!-- post $orig_title needs update : $post_needs_update -->"; db_query($link, "UPDATE ttrss_entries\n\t\t\t\t\t\t\tSET title = '{$entry_title}', content = '{$entry_content}',\n\t\t\t\t\t\t\t\tcontent_hash = '{$content_hash}',\n\t\t\t\t\t\t\t\tupdated = '{$entry_timestamp_fmt}',\n\t\t\t\t\t\t\t\tnum_comments = '{$num_comments}'\n\t\t\t\t\t\t\tWHERE id = '{$ref_id}'"); if (!$update_insignificant) { if ($mark_unread_on_update) { db_query($link, "UPDATE ttrss_user_entries\n\t\t\t\t\t\t\t\t\tSET last_read = null, unread = true WHERE ref_id = '{$ref_id}'"); } else { if ($update_on_checksum_change) { db_query($link, "UPDATE ttrss_user_entries\n\t\t\t\t\t\t\t\t\tSET last_read = null WHERE ref_id = '{$ref_id}'\n\t\t\t\t\t\t\t\t\t\tAND unread = false"); } } } } } db_query($link, "COMMIT"); if ($debug_enabled) { _debug("update_rss_feed: assigning labels..."); } assign_article_to_labels($link, $entry_ref_id, $article_filters, $owner_uid); if ($debug_enabled) { _debug("update_rss_feed: looking for enclosures..."); } // enclosures $enclosures = array(); if ($use_simplepie) { $encs = $item->get_enclosures(); if (is_array($encs)) { foreach ($encs as $e) { $e_item = array($e->link, $e->type, $e->length); array_push($enclosures, $e_item); } } } else { // <enclosure> $e_ctr = $item['enclosure#']; if ($e_ctr > 0) { $e_item = array($item['enclosure@url'], $item['enclosure@type'], $item['enclosure@length']); array_push($enclosures, $e_item); for ($i = 0; $i <= $e_ctr; $i++) { if ($item["enclosure#{$i}@url"]) { $e_item = array($item["enclosure#{$i}@url"], $item["enclosure#{$i}@type"], $item["enclosure#{$i}@length"]); array_push($enclosures, $e_item); } } } // <media:content> // can there be many of those? yes -fox $m_ctr = $item['media']['content#']; if ($m_ctr > 0) { $e_item = array($item['media']['content@url'], $item['media']['content@medium'], $item['media']['content@length']); array_push($enclosures, $e_item); for ($i = 0; $i <= $m_ctr; $i++) { if ($item["media"]["content#{$i}@url"]) { $e_item = array($item["media"]["content#{$i}@url"], $item["media"]["content#{$i}@medium"], $item["media"]["content#{$i}@length"]); array_push($enclosures, $e_item); } } } } if ($debug_enabled) { _debug("update_rss_feed: article enclosures:"); print_r($enclosures); } db_query($link, "BEGIN"); foreach ($enclosures as $enc) { $enc_url = db_escape_string($enc[0]); $enc_type = db_escape_string($enc[1]); $enc_dur = db_escape_string($enc[2]); $result = db_query($link, "SELECT id FROM ttrss_enclosures\n\t\t\t\t\t\tWHERE content_url = '{$enc_url}' AND post_id = '{$entry_ref_id}'"); if (db_num_rows($result) == 0) { db_query($link, "INSERT INTO ttrss_enclosures\n\t\t\t\t\t\t\t(content_url, content_type, title, duration, post_id) VALUES\n\t\t\t\t\t\t\t('{$enc_url}', '{$enc_type}', '', '{$enc_dur}', '{$entry_ref_id}')"); } } db_query($link, "COMMIT"); // check for manual tags (we have to do it here since they're loaded from filters) foreach ($article_filters as $f) { if ($f["type"] == "tag") { $manual_tags = trim_array(explode(",", $f["param"])); foreach ($manual_tags as $tag) { if (tag_is_valid($tag)) { array_push($entry_tags, $tag); } } } } // Skip boring tags $boring_tags = trim_array(explode(",", mb_strtolower(get_pref($link, 'BLACKLISTED_TAGS', $owner_uid, ''), 'utf-8'))); $filtered_tags = array(); $tags_to_cache = array(); if ($entry_tags && is_array($entry_tags)) { foreach ($entry_tags as $tag) { if (array_search($tag, $boring_tags) === false) { array_push($filtered_tags, $tag); } } } $filtered_tags = array_unique($filtered_tags); if ($debug_enabled) { _debug("update_rss_feed: filtered article tags:"); print_r($filtered_tags); } // Save article tags in the database if (count($filtered_tags) > 0) { db_query($link, "BEGIN"); foreach ($filtered_tags as $tag) { $tag = sanitize_tag($tag); $tag = db_escape_string($tag); if (!tag_is_valid($tag)) { continue; } $result = db_query($link, "SELECT id FROM ttrss_tags\n\t\t\t\t\t\t\tWHERE tag_name = '{$tag}' AND post_int_id = '{$entry_int_id}' AND\n\t\t\t\t\t\t\towner_uid = '{$owner_uid}' LIMIT 1"); if ($result && db_num_rows($result) == 0) { db_query($link, "INSERT INTO ttrss_tags\n\t\t\t\t\t\t\t\t\t(owner_uid,tag_name,post_int_id)\n\t\t\t\t\t\t\t\t\tVALUES ('{$owner_uid}','{$tag}', '{$entry_int_id}')"); } array_push($tags_to_cache, $tag); } /* update the cache */ $tags_to_cache = array_unique($tags_to_cache); $tags_str = db_escape_string(join(",", $tags_to_cache)); db_query($link, "UPDATE ttrss_user_entries\n\t\t\t\t\t\tSET tag_cache = '{$tags_str}' WHERE ref_id = '{$entry_ref_id}'\n\t\t\t\t\t\tAND owner_uid = {$owner_uid}"); db_query($link, "COMMIT"); } if ($debug_enabled) { _debug("update_rss_feed: article processed"); } } if (!$last_updated) { if ($debug_enabled) { _debug("update_rss_feed: new feed, catching it up..."); } catchup_feed($link, $feed, false, $owner_uid); } if ($debug_enabled) { _debug("purging feed..."); } purge_feed($link, $feed, 0, $debug_enabled); db_query($link, "UPDATE ttrss_feeds\n\t\t\t\tSET last_updated = NOW(), last_error = '' WHERE id = '{$feed}'"); // db_query($link, "COMMIT"); } else { if ($use_simplepie) { $error_msg = mb_substr($rss->error(), 0, 250); } else { $error_msg = mb_substr(magpie_error(), 0, 250); } if ($debug_enabled) { _debug("update_rss_feed: error fetching feed: {$error_msg}"); } $error_msg = db_escape_string($error_msg); db_query($link, "UPDATE ttrss_feeds SET last_error = '{$error_msg}',\n\t\t\t\t\tlast_updated = NOW() WHERE id = '{$feed}'"); } if ($use_simplepie) { unset($rss); } if ($debug_enabled) { _debug("update_rss_feed: done"); } }
/** * Processing the parameters into placeholders * @param string $spie snippet parameters * @return array placeholders */ private function _setSimplePieModxPlaceholders($spie) { /** * @link http://github.com/simplepie/simplepie/tree/one-dot-two */ if (!file_exists($spie['simplePieClassFile'])) { return 'File ' . $spie['simplePieClassFile'] . ' does not exist.'; } include_once $spie['simplePieClassFile']; $feed = new SimplePie(); $joinKey = 0; foreach ($spie['setFeedUrl'] as $setFeedUrl) { $feed->set_cache_location($spie['setCacheLocation']); $feed->set_feed_url($setFeedUrl); if (isset($spie['setInputEncoding'])) { $feed->set_input_encoding($spie['setInputEncoding']); } if (isset($spie['setOutputEncoding'])) { $feed->set_output_encoding($spie['setOutputEncoding']); } // if no cURL, try fsockopen if (isset($spie['forceFSockopen'])) { $feed->force_fsockopen(true); } if (isset($spie['enableCache'])) $feed->enable_cache($spie['enableCache']); if (isset($spie['enableOrderByDate'])) $feed->enable_order_by_date($spie['enableOrderByDate']); if (isset($spie['setCacheDuration'])) $feed->set_cache_duration($spie['setCacheDuration']); if (!empty($spie['setFaviconHandler'])) $feed->set_favicon_handler($spie['setFaviconHandler'][0], $spie['setFaviconHandler'][1]); if (!empty($spie['setImageHandler'])) { // handler_image.php?image=67d5fa9a87bad230fb03ea68b9f71090 $feed->set_image_handler($spie['setImageHandler'][0], $spie['setImageHandler'][1]); } // disabled since these are all splitted into a single fetching // it's been used with different way, see below looping // if (isset($spie['setItemLimit'])) // $feed->set_item_limit((int) $spie['setItemLimit']); if (isset($spie['setJavascript'])) $feed->set_javascript($spie['setJavascript']); if (isset($spie['stripAttributes'])) $feed->strip_attributes(array_merge($feed->strip_attributes, $spie['stripAttributes'])); if (isset($spie['stripComments'])) $feed->strip_comments($spie['stripComments']); if (isset($spie['stripHtmlTags'])) $feed->strip_htmltags(array_merge($feed->strip_htmltags, $spie['stripHtmlTags'])); /** * Initiating the Feeding. * This always be placed AFTER all the settings above. */ if (!$feed->init()) { echo $feed->error(); return FALSE; } $countItems = count($feed->get_items()); if (1 > $countItems) { continue; } $feed->handle_content_type(); $countLimit = 0; foreach ($feed->get_items($getItemStart, $getItemEnd) as $item) { if (isset($spie['setItemLimit']) && $spie['setItemLimit'] == $countLimit) continue; $phArray[$joinKey]['favicon'] = $feed->get_favicon(); $phArray[$joinKey]['link'] = $item->get_link(); $phArray[$joinKey]['title'] = $item->get_title(); $phArray[$joinKey]['description'] = $item->get_description(); $phArray[$joinKey]['content'] = $item->get_content(); $phArray[$joinKey]['permalink'] = $item->get_permalink(); $parsedUrl = parse_url($phArray[$joinKey]['permalink']); $implodedParsedUrl = $parsedUrl['scheme'] . '://' . $parsedUrl['host']; $imageLink = $feed->get_image_link() != '' ? $feed->get_image_link() : $implodedParsedUrl; $phArray[$joinKey]['imageLink'] = $imageLink; $phArray[$joinKey]['imageTitle'] = $feed->get_image_title(); $phArray[$joinKey]['imageUrl'] = $feed->get_image_url(); $phArray[$joinKey]['imageWidth'] = $feed->get_image_width(); $phArray[$joinKey]['imageHeight'] = $feed->get_image_height(); $phArray[$joinKey]['date'] = $item->get_date($spie['dateFormat']); $phArray[$joinKey]['localDate'] = $item->get_local_date($spie['localDateFormat']); $phArray[$joinKey]['copyright'] = $item->get_copyright(); $phArray[$joinKey]['latitude'] = $feed->get_latitude(); $phArray[$joinKey]['longitude'] = $feed->get_longitude(); $phArray[$joinKey]['language'] = $feed->get_language(); $phArray[$joinKey]['encoding'] = $feed->get_encoding(); if ($item->get_authors()) { foreach ($item->get_authors() as $authorObject) { $authorName = $authorObject->get_name(); $authorLink = $authorObject->get_link(); $authorEmail = $authorObject->get_email(); } $phArray[$joinKey]['authorName'] = $authorName; $phArray[$joinKey]['authorLink'] = $authorLink; $phArray[$joinKey]['authorEmail'] = $authorEmail; } $category = $item->get_category(); if ($category) { $phArray[$joinKey]['category'] = htmlspecialchars_decode($category->get_label(), ENT_QUOTES); } $contributor = $item->get_contributor(); $phArray[$joinKey]['contributor'] = ''; if ($contributor) { $phArray[$joinKey]['contributor'] = $contributor->get_name(); } if ($feed->get_type() & SIMPLEPIE_TYPE_NONE) { $phArray[$joinKey]['getType'] = 'Unknown'; } elseif ($feed->get_type() & SIMPLEPIE_TYPE_RSS_ALL) { $phArray[$joinKey]['getType'] = 'RSS'; } elseif ($feed->get_type() & SIMPLEPIE_TYPE_ATOM_ALL) { $phArray[$joinKey]['getType'] = 'Atom'; } elseif ($feed->get_type() & SIMPLEPIE_TYPE_ALL) { $phArray[$joinKey]['getType'] = 'Supported'; } // Media from Flickr RSS stream if ($enclosure = $item->get_enclosure()) { $phArray[$joinKey]['itemImageThumbnailUrl'] = $enclosure->get_thumbnail(); $phArray[$joinKey]['itemImageWidth'] = $enclosure->get_width(); $phArray[$joinKey]['itemImageHeight'] = $enclosure->get_height(); } $countLimit++; $joinKey++; } // foreach ($feed->get_items($getItemStart, $getItemEnd) as $item) } // foreach ($spie['setFeedUrl'] as $setFeedUrl) return $this->_filterModxTags($phArray); }