This tells SimplePie to ignore the content-type provided by the server.
Be careful when using this option, as it will also disable autodiscovery.
public force_feed ( boolean $enable = false ) | ||
$enable | boolean | Force the given data/URL to be treated as a feed |
/** * Get a parsed XML Feed Source * * @param string $url Url for feed source. * @param integer $cache_time Time to cache feed for (using internal cache mechanism). * * @return mixed SimplePie parsed object on success, false on failure. * * @since 12.2 * @deprecated 4.0 Use JFeedFactory($url) instead. * * @note In 3.2 will be proxied to JFeedFactory() */ public static function getFeedParser($url, $cache_time = 0) { JLog::add(__METHOD__ . ' is deprecated. Use JFeedFactory() or supply Simple Pie instead.', JLog::WARNING, 'deprecated'); $cache = JFactory::getCache('feed_parser', 'callback'); if ($cache_time > 0) { $cache->setLifeTime($cache_time); } $simplepie = new SimplePie(null, null, 0); $simplepie->enable_cache(false); $simplepie->set_feed_url($url); $simplepie->force_feed(true); $contents = $cache->get(array($simplepie, 'init'), null, false, false); if ($contents) { return $simplepie; } JLog::add(JText::_('JLIB_UTIL_ERROR_LOADING_FEED_DATA'), JLog::WARNING, 'jerror'); return false; }
function show_news($newsfeed, $newsitems, $newsprefiximage, $newssuffiximage) { $rss = new SimplePie($newsfeed, dirname(__FILE__) . '/cache'); $rss->force_feed(true); // MAKE IT WORK $rss->set_cache_duration(60 * 5); $rss->init(); $rss->handle_content_type(); // if ($rss->error()) { // echo htmlentities($rss->error()); // return; // } foreach ($rss->get_items(0, $newsitems) as $item) { $title = $item->get_title(); $url = $item->get_permalink(); if ($newsprefiximage != '') { echo "<img src='{$newsprefiximage}'>"; } echo "<a href={$url}>{$title}</a>"; if ($newssuffiximage != '') { echo "<img src='{$newssuffiximage}'>"; } echo "<br />"; } }
function fetch_feed2($url) { require_once ABSPATH . WPINC . '/class-feed.php'; $feed = new SimplePie(); $feed->set_sanitize_class('WP_SimplePie_Sanitize_KSES'); // We must manually overwrite $feed->sanitize because SimplePie's // constructor sets it before we have a chance to set the sanitization class $feed->sanitize = new WP_SimplePie_Sanitize_KSES(); $feed->set_cache_class('WP_Feed_Cache'); $feed->set_file_class('WP_SimplePie_File'); $feed->set_feed_url($url); $feed->force_feed(true); /** This filter is documented in wp-includes/class-feed.php */ $feed->set_cache_duration(apply_filters('wp_feed_cache_transient_lifetime', 12 * HOUR_IN_SECONDS, $url)); /** * Fires just before processing the SimplePie feed object. * * @since 3.0.0 * * @param object &$feed SimplePie feed object, passed by reference. * @param mixed $url URL of feed to retrieve. If an array of URLs, the feeds are merged. */ do_action_ref_array('wp_feed_options', array(&$feed, $url)); $feed->init(); $feed->handle_content_type(); if ($feed->error()) { return new WP_Error('simplepie-error', $feed->error()); } return $feed; }
function getFeeds() { $app = JFactory::getApplication(); $params = JComponentHelper::getParams('com_jce'); $limit = $params->get('feed_limit', 2); $feeds = array(); $options = array('rssUrl' => 'http://www.joomlacontenteditor.net/news/feed/rss/latest-news?format=feed', 'cache_time' => $params->get('feed_cachetime', 86400)); // use this directly instead of JFactory::getXMLParserto avoid the feed data error jimport('simplepie.simplepie'); if (!is_writable(JPATH_BASE . '/cache')) { $options['cache_time'] = 0; } $rss = new SimplePie($options['rssUrl'], JPATH_BASE . '/cache', isset($options['cache_time']) ? $options['cache_time'] : 0); $rss->force_feed(true); $rss->handle_content_type(); if ($rss->init()) { $count = $rss->get_item_quantity(); if ($count) { $count = $count > $limit ? $limit : $count; for ($i = 0; $i < $count; $i++) { $feed = new StdClass(); $item = $rss->get_item($i); $feed->link = $item->get_link(); $feed->title = $item->get_title(); $feed->description = $item->get_description(); $feeds[] = $feed; } } } return $feeds; }
/** * Register the service provider. * * @return void */ public function register() { $this->app['feed.parser'] = function ($app) { $cache = PATH_APP . DS . 'cache'; $cache .= DS . (isset($app['client']->alias) ? $app['client']->alias : $app['client']->name); include_once PATH_CORE . DS . 'libraries' . DS . 'simplepie' . DS . 'simplepie.php'; $reader = new \SimplePie(null, $cache, $app['config']->get('cachetime', 15)); $reader->enable_cache(false); $reader->force_feed(true); return $reader; }; }
protected function fetchSite($site) { $feed = new \SimplePie(); $feed->force_feed(true); $feed->set_item_limit(20); $feed->set_feed_url($site); $feed->enable_cache(false); $feed->set_output_encoding('utf-8'); $feed->init(); foreach ($feed->get_items() as $item) { $this->outputItem(['site' => $site, 'title' => $item->get_title(), 'link' => $item->get_permalink(), 'date' => new \Carbon\Carbon($item->get_date()), 'content' => $item->get_content()]); } }
private function getFeedByUrl($url) { // Create a new instance of the SimplePie object $feed = new \SimplePie(); // Set feed $feed->set_feed_url($url); // Allow us to change the input encoding from the URL string if we want to. (optional) if (!empty($_GET['input'])) { $feed->set_input_encoding($_GET['input']); } // Allow us to choose to not re-order the items by date. (optional) if (!empty($_GET['orderbydate']) && $_GET['orderbydate'] == 'false') { $feed->enable_order_by_date(false); } // Trigger force-feed if (!empty($_GET['force']) && $_GET['force'] == 'true') { $feed->force_feed(true); } $feed->set_cache_location(dirname(__FILE__) . '/../cache/'); $feed->init(); return $feed->get_items(0, 10); }
/** * Opens an RSS feed, parses and loads the contents. * * @param string $url The URL of the RSS feed * @param bool $nativeOrder If true, disable order by date to preserve native ordering * @param bool $force Force SimplePie to parse the feed despite errors * * @return object An object that encapsulates the feed contents and operations on those contents. * @throws FeedParserException If opening or parsing feed fails **/ public function parseFeed($url, $nativeOrder = false, $force = false) { require_once PATH_SYSTEM . '/vendors/SimplePie.php'; $feed = new SimplePie(); $feed->set_timeout($this->timeout); $feed->set_feed_url($url); $feed->enable_order_by_date(!$nativeOrder); $feed->force_feed($force); if ($this->cacheDuration != null) { $feed->set_cache_duration(intval($this->cacheDuration)); } if ($this->cacheDirectory != null) { $feed->set_cache_location($this->cacheDirectory); } if ($this->stripHtmlTags != null) { $feed->strip_htmltags($this->stripHtmlTags); } @$feed->init(); if ($err = $feed->error()) { throw new FeedParserException($err); } return $feed; }
function fetch($url, $force_feed = true) { $feed = new SimplePie(); $feed->set_feed_url($url); if (version_compare(SIMPLEPIE_VERSION, '1.3-dev', '>')) { $feed->set_cache_location('wp-transient'); $feed->registry->register('Cache', 'WP_Feed_Cache_Transient'); $feed->registry->register('File', 'FeedWordPress_File'); } else { $feed->set_cache_class('WP_Feed_Cache'); $feed->set_file_class('FeedWordPress_File'); } $feed->set_content_type_sniffer_class('FeedWordPress_Content_Type_Sniffer'); $feed->set_file_class('FeedWordPress_File'); $feed->force_feed($force_feed); $feed->set_cache_duration(FeedWordPress::cache_duration()); $feed->init(); $feed->handle_content_type(); if ($feed->error()) { $ret = new WP_Error('simplepie-error', $feed->error()); } else { $ret = $feed; } return $ret; }
/** * Get an parsed XML Feed Source * * @access public * @param string Url for feed source * @param int Time to cache feed * * @return mixed SimplePie parsed object on success, false on failure * @since: Nooku Server 0.7 */ function getFeedParser($url, $cache_time = null) { jimport('simplepie.simplepie'); $simplepie = new SimplePie(); $simplepie->set_feed_url($url); $cache_path = JFactory::getConfig()->getValue('config.cache_path', JPATH_ROOT . DS . 'cache'); if (is_writable($cache_path) && JFactory::getConfig()->getValue('config.caching')) { if (is_null($cache_time)) { $cache_time = JFactory::getConfig()->getValue('config.caching') * 60; } $simplepie->set_cache_duration($cache_time); $simplepie->set_cache_location($cache_path); } else { $simplepie->enable_cache(false); } $simplepie->force_feed(true); $simplepie->handle_content_type(); if ($simplepie->init()) { return $simplepie; } JError::raiseWarning('SOME_ERROR_CODE', JText::_('ERROR LOADING FEED DATA')); return false; }
/** * A clone of the function 'fetch_feed' in wp-includes/feed.php [line #529] * * Called from 'wprss_get_feed_items' * * @since 3.5 */ function wprss_fetch_feed($url, $source = NULL) { // Import SimplePie require_once ABSPATH . WPINC . '/class-feed.php'; // Trim the URL $url = trim($url); // Initialize the Feed $feed = new SimplePie(); // Obselete method calls ? //$feed->set_cache_class( 'WP_Feed_Cache' ); //$feed->set_file_class( 'WP_SimplePie_File' ); $feed->set_feed_url($url); $feed->set_autodiscovery_level(SIMPLEPIE_LOCATOR_ALL); // If a feed source was passed if ($source !== NULL) { // Get the force feed option for the feed source $force_feed = get_post_meta($source, 'wprss_force_feed', TRUE); // If turned on, force the feed if ($force_feed == 'true') { $feed->force_feed(TRUE); } } // Set timeout to 30s. Default: 15s $feed->set_timeout(30); //$feed->set_cache_duration( apply_filters( 'wp_feed_cache_transient_lifetime', 12 * HOUR_IN_SECONDS, $url ) ); $feed->enable_cache(FALSE); // Reference array action hook, for the feed object and the URL do_action_ref_array('wp_feed_options', array(&$feed, $url)); // Prepare the tags to strip from the feed $tags_to_strip = apply_filters('wprss_feed_tags_to_strip', $feed->strip_htmltags, $source); // Strip them $feed->strip_htmltags($tags_to_strip); // Fetch the feed $feed->init(); $feed->handle_content_type(); // Convert the feed error into a WP_Error, if applicable if ($feed->error()) { return new WP_Error('simplepie-error', $feed->error()); } // If no error, return the feed return $feed; }
/** * * @param object $bookmark * @param object $owner */ protected function do_rss($bookmark, $owner) { // no bookmark, no fun if (empty($bookmark) || !is_object($bookmark)) { return false; } // no owner means no email, so no reason to parse if (empty($owner) || !is_object($owner)) { return false; } // instead of the way too simple fetch_feed, we'll use SimplePie itself if (!class_exists('SimplePie')) { require_once ABSPATH . WPINC . '/class-simplepie.php'; } $url = htmlspecialchars_decode($bookmark->link_rss); $last_updated = strtotime($bookmark->link_updated); static::debug('Fetching: ' . $url, 6); $feed = new SimplePie(); $feed->set_feed_url($url); $feed->set_cache_duration(static::revisit_time - 10); $feed->set_cache_location($this->cachedir); $feed->force_feed(true); // optimization $feed->enable_order_by_date(true); $feed->remove_div(true); $feed->strip_comments(true); $feed->strip_htmltags(false); $feed->strip_attributes(true); $feed->set_image_handler(false); $feed->init(); $feed->handle_content_type(); if ($feed->error()) { $err = new WP_Error('simplepie-error', $feed->error()); static::debug('Error: ' . $err->get_error_message(), 4); $this->failed($owner->user_email, $url, $err->get_error_message()); return $err; } // set max items to 12 // especially useful with first runs $maxitems = $feed->get_item_quantity(12); $feed_items = $feed->get_items(0, $maxitems); $feed_title = $feed->get_title(); // set the link name from the RSS title if (!empty($feed_title) && $bookmark->link_name != $feed_title) { global $wpdb; $wpdb->update($wpdb->prefix . 'links', array('link_name' => $feed_title), array('link_id' => $bookmark->link_id)); } // if there's a feed author, get it, we may need it if there's no entry // author $feed_author = $feed->get_author(); $last_updated_ = 0; if ($maxitems > 0) { foreach ($feed_items as $item) { // U stands for Unix Time $date = $item->get_date('U'); if ($date > $last_updated) { $fromname = $feed_title; $author = $item->get_author(); if ($author) { $fromname = $fromname . ': ' . $author->get_name(); } elseif ($feed_author) { $fromname = $fromname . ': ' . $feed_author->get_name(); } // this is to set the sender mail from our own domain $frommail = get_user_meta($owner->ID, 'blogroll2email_email', true); if (!$frommail) { $sitedomain = parse_url(get_bloginfo('url'), PHP_URL_HOST); $frommail = static::schedule . '@' . $sitedomain; } $from = $fromname . '<' . $frommail . '>'; $content = $item->get_content(); $matches = array(); preg_match_all('/farm[0-9]\\.staticflickr\\.com\\/[0-9]+\\/([0-9]+_[0-9a-zA-Z]+_m\\.jpg)/s', $content, $matches); if (!empty($matches[0])) { foreach ($matches[0] as $to_replace) { $clean = str_replace('_m.jpg', '_c.jpg', $to_replace); $content = str_replace($to_replace, $clean, $content); } $content = preg_replace("/(width|height)=\"(.*?)\" ?/is", '', $content); } $content = apply_filters('blogroll2email_message', $content); if ($this->send($owner->user_email, $item->get_link(), $item->get_title(), $from, $url, $item->get_content(), $date)) { if ($date > $last_updated_) { $last_updated_ = $date; } } } } } // poke the link's last update field, so we know what was the last sent // entry's date $this->update_link_date($bookmark, $last_updated_); }
function renderVersionStatusReport(&$needsupdate) { jimport("joomla.filesystem.folder"); if (JEVHelper::isAdminUser()) { // get RSS parsed object $options = array(); $rssUrl = 'https://www.jevents.net/versions30.xml'; $cache_time = 86400; error_reporting(0); ini_set('display_errors', 0); jimport('simplepie.simplepie'); // this caching doesn't work!!! //$cache = JFactory::getCache('feed_parser', 'callback'); //$cache->setLifeTime($cache_time); //$cache->setCaching(true); $rssDoc = new SimplePie(null, null, 0); $rssDoc->enable_cache(false); $rssDoc->set_feed_url($rssUrl); $rssDoc->force_feed(true); $rssDoc->set_item_limit(999); //$results = $cache->get(array($rssDoc, 'init'), null, false, false); $results = $rssDoc->init(); if ($results == false) { return false; } else { $this->generateVersionsFile($rssDoc); $rows = array(); $items = $rssDoc->get_items(); foreach ($items as $item) { $apps = array(); if (strpos($item->get_title(), "layout_") === 0) { $layout = str_replace("layout_", "", $item->get_title()); if (JFolder::exists(JEV_PATH . "views/{$layout}")) { // club layouts $xmlfiles1 = JFolder::files(JEV_PATH . "views/{$layout}", "manifest\\.xml", true, true); if ($xmlfiles1 && count($xmlfiles1) > 0) { foreach ($xmlfiles1 as $manifest) { if (realpath($manifest) != $manifest) { continue; } if (!($manifestdata = $this->getValidManifestFile($manifest))) { continue; } $app = new stdClass(); $app->name = $manifestdata["name"]; $app->version = $manifestdata["version"]; $apps["layout_" . basename(dirname($manifest))] = $app; } } } // package version if (JFolder::exists(JPATH_ADMINISTRATOR . "/manifests/files")) { // club layouts $xmlfiles1 = JFolder::files(JPATH_ADMINISTRATOR . "/manifests/files", "{$layout}\\.xml", true, true); if (!$xmlfiles1) { continue; } foreach ($xmlfiles1 as $manifest) { if (realpath($manifest) != $manifest) { continue; } if (!($manifestdata = $this->getValidManifestFile($manifest))) { continue; } $app = new stdClass(); $app->name = $manifestdata["name"]; $app->version = $manifestdata["version"]; $apps["layout_" . basename(dirname($manifest))] = $app; } } } else { if (strpos($item->get_title(), "module_") === 0) { $module = str_replace("module_", "", $item->get_title()); // modules if (JFolder::exists(JPATH_SITE . "/modules/{$module}")) { $xmlfiles1 = JFolder::files(JPATH_SITE . "/modules/{$module}", "\\.xml", true, true); if (!$xmlfiles1) { continue; } foreach ($xmlfiles1 as $manifest) { if (realpath($manifest) != $manifest) { continue; } if (!($manifestdata = $this->getValidManifestFile($manifest))) { continue; } $app = new stdClass(); $app->name = $manifestdata["name"]; $app->version = $manifestdata["version"]; $name = "module_" . str_replace(".xml", "", basename($manifest)); $apps[$name] = $app; } } } else { if (strpos($item->get_title(), "plugin_") === 0) { $plugin = explode("_", str_replace("plugin_", "", $item->get_title()), 2); if (count($plugin) < 2) { continue; } // plugins if (JFolder::exists(JPATH_SITE . "/plugins/" . $plugin[0] . "/" . $plugin[1])) { // plugins $xmlfiles1 = JFolder::files(JPATH_SITE . "/plugins/" . $plugin[0] . "/" . $plugin[1], "\\.xml", true, true); foreach ($xmlfiles1 as $manifest) { if (!($manifestdata = $this->getValidManifestFile($manifest))) { continue; } $app = new stdClass(); $app->name = $manifestdata["name"]; $app->version = $manifestdata["version"]; $name = str_replace(".xml", "", basename($manifest)); $name = "plugin_" . basename(dirname(dirname($manifest))) . "_" . $name; $apps[$name] = $app; } } } else { if (strpos($item->get_title(), "component_") === 0) { $component = str_replace("component_", "", $item->get_title()); if (JFolder::exists(JPATH_ADMINISTRATOR . "/components/" . $component)) { // modules $xmlfiles1 = JFolder::files(JPATH_ADMINISTRATOR . "/components/" . $component, "\\.xml", true, true); if (!$xmlfiles1) { continue; } foreach ($xmlfiles1 as $manifest) { if (!($manifestdata = $this->getValidManifestFile($manifest))) { continue; } $app = new stdClass(); $app->name = $manifestdata["name"]; $app->version = $manifestdata["version"]; $name = "component_" . basename(dirname($manifest)); $apps[$name] = $app; } } } else { continue; } } } } foreach ($apps as $appname => $app) { $iteminfo = json_decode($item->get_description()); if (version_compare($app->version, $iteminfo->version, "<")) { $link = $iteminfo->link != "" ? "<a href='" . $iteminfo->link . "' target='_blank'>" . $app->name . "</a>" : $app->name; if ($iteminfo->criticalversion != "" && version_compare($app->version, $iteminfo->criticalversion, "<")) { $rows[] = array($link, $appname, $app->version, $iteminfo->version, "<strong>" . $iteminfo->criticalversion . "</strong>"); } else { $rows[] = array($link, $appname, $app->version, $iteminfo->version, ""); } } } } if (count($rows) > 0) { $output = '<table class="versionstatuslist"><tr>'; $output .= '<th>' . JText::_("JEV_APPNAME") . '</th>'; $output .= '<th>' . JText::_("JEV_APPCODE") . '</th>'; $output .= '<th>' . JText::_("JEV_CURRENTVERSION") . '</th>'; $output .= '<th>' . JText::_("JEV_LATESTVERSION") . '</th>'; $output .= '<th>' . JText::_("JEV_CRITICALVERSION") . '</th>'; $output .= '</tr>'; $k = 0; foreach ($rows as $row) { $output .= '<tr class="row' . $k . '"><td>'; $output .= implode("</td><td>", $row); $output .= '</td></tr>'; $k = ($k + 1) % 2; } $output .= '</table>'; $needsupdate = true; return $output; } } } return false; }
/** * Get a specific RSS feed * * @param $url string/array URL of the feed or array of URL * @param $cache_duration timestamp cache duration (default DAY_TIMESTAMP) * * @return feed object **/ static function getRSSFeed($url, $cache_duration = DAY_TIMESTAMP) { global $CFG_GLPI; $feed = new SimplePie(); $feed->set_cache_location(GLPI_RSS_DIR); $feed->set_cache_duration($cache_duration); // proxy support if (!empty($CFG_GLPI["proxy_name"])) { $prx_opt = array(); $prx_opt[CURLOPT_PROXY] = $CFG_GLPI["proxy_name"]; $prx_opt[CURLOPT_PROXYPORT] = $CFG_GLPI["proxy_port"]; if (!empty($CFG_GLPI["proxy_user"])) { $prx_opt[CURLOPT_HTTPAUTH] = CURLAUTH_ANYSAFE; $prx_opt[CURLOPT_PROXYUSERPWD] = $CFG_GLPI["proxy_user"] . ":" . Toolbox::decrypt($CFG_GLPI["proxy_passwd"], GLPIKEY); } $feed->set_curl_options($prx_opt); } $feed->enable_cache(true); $feed->set_feed_url($url); $feed->force_feed(true); // Initialize the whole SimplePie object. Read the feed, process it, parse it, cache it, and // all that other good stuff. The feed's information will not be available to SimplePie before // this is called. $feed->init(); // We'll make sure that the right content type and character encoding gets set automatically. // This function will grab the proper character encoding, as well as set the content type to text/html. $feed->handle_content_type(); if ($feed->error()) { return false; } return $feed; }
public function items() { if ($this->input->is_ajax_request()) { $this->readerself_library->set_template('_json'); $this->readerself_library->set_content_type('application/json'); $content = array(); } else { $this->readerself_library->set_template('_plain'); $this->readerself_library->set_content_type('text/plain'); $content = ''; } if ($this->input->is_cli_request() && !$this->config->item('refresh_by_cron')) { $content .= 'Refresh by cron disabled' . "\n"; } else { include_once 'thirdparty/simplepie/autoloader.php'; include_once 'thirdparty/simplepie/idn/idna_convert.class.php'; if ($this->config->item('facebook/enabled')) { include_once 'thirdparty/facebook/autoload.php'; $fb = new Facebook\Facebook(array('app_id' => $this->config->item('facebook/id'), 'app_secret' => $this->config->item('facebook/secret'))); $fbApp = $fb->getApp(); $accessToken = $fbApp->getAccessToken(); } $query = $this->db->query('SELECT fed.* FROM ' . $this->db->dbprefix('feeds') . ' AS fed WHERE fed.fed_nextcrawl IS NULL OR fed.fed_nextcrawl <= ? GROUP BY fed.fed_id HAVING (SELECT COUNT(DISTINCT(sub.mbr_id)) FROM ' . $this->db->dbprefix('subscriptions') . ' AS sub WHERE sub.fed_id = fed.fed_id) > 0', array(date('Y-m-d H:i:s'))); if ($query->num_rows() > 0) { $microtime_start = microtime(1); $errors = 0; foreach ($query->result() as $fed) { $parse_url = parse_url($fed->fed_link); if (isset($parse_url['host']) == 1 && $parse_url['host'] == 'www.facebook.com' && $this->config->item('facebook/enabled')) { try { $parts = explode('/', $parse_url['path']); $total_parts = count($parts); $last_part = $parts[$total_parts - 1]; $request = new Facebook\FacebookRequest($fbApp, $accessToken, 'GET', $last_part . '?fields=link,name,about'); $response = $fb->getClient()->sendRequest($request); $result = $response->getDecodedBody(); $request = new Facebook\FacebookRequest($fbApp, $accessToken, 'GET', $last_part . '?fields=feed{created_time,id,message,story,full_picture,place,type,status_type,link}'); $response = $fb->getClient()->sendRequest($request); $posts = $response->getDecodedBody(); $this->readerself_library->crawl_items_facebook($fed->fed_id, $posts['feed']['data']); $lastitem = $this->db->query('SELECT itm.itm_datecreated FROM ' . $this->db->dbprefix('items') . ' AS itm WHERE itm.fed_id = ? GROUP BY itm.itm_id ORDER BY itm.itm_id DESC LIMIT 0,1', array($fed->fed_id))->row(); $this->db->set('fed_title', $result['name']); $this->db->set('fed_url', $result['link']); $this->db->set('fed_link', $result['link']); if (isset($parse_url['host']) == 1) { $this->db->set('fed_host', $parse_url['host']); } $this->db->set('fed_description', $result['about']); $this->db->set('fed_lasterror', ''); $this->db->set('fed_lastcrawl', date('Y-m-d H:i:s')); if ($lastitem) { $nextcrawl = ''; //older than 96 hours, next crawl in 12 hours if ($lastitem->itm_datecreated < date('Y-m-d H:i:s', time() - 3600 * 24 * 96)) { $nextcrawl = date('Y-m-d H:i:s', time() + 3600 * 12); //older than 48 hours, next crawl in 6 hours } else { if ($lastitem->itm_datecreated < date('Y-m-d H:i:s', time() - 3600 * 48)) { $nextcrawl = date('Y-m-d H:i:s', time() + 3600 * 6); //older than 24 hours, next crawl in 3 hours } else { if ($lastitem->itm_datecreated < date('Y-m-d H:i:s', time() - 3600 * 24)) { $nextcrawl = date('Y-m-d H:i:s', time() + 3600 * 3); } } } $this->db->set('fed_nextcrawl', $nextcrawl); } $this->db->where('fed_id', $fed->fed_id); $this->db->update('feeds'); } catch (Facebook\Exceptions\FacebookResponseException $e) { $errors++; $this->db->set('fed_lasterror', 'Graph returned an error: ' . $e->getMessage()); $this->db->set('fed_lastcrawl', date('Y-m-d H:i:s')); $this->db->where('fed_id', $fed->fed_id); $this->db->update('feeds'); } catch (Facebook\Exceptions\FacebookSDKException $e) { $errors++; $this->db->set('fed_lasterror', 'Facebook SDK returned an error: ' . $e->getMessage()); $this->db->set('fed_lastcrawl', date('Y-m-d H:i:s')); $this->db->where('fed_id', $fed->fed_id); $this->db->update('feeds'); } } else { $sp_feed = new SimplePie(); $sp_feed->set_feed_url(convert_to_ascii($fed->fed_link)); $sp_feed->enable_cache(false); $sp_feed->set_timeout(5); $sp_feed->force_feed(true); $sp_feed->init(); $sp_feed->handle_content_type(); if ($sp_feed->error()) { $errors++; $this->db->set('fed_lasterror', $sp_feed->error()); $this->db->set('fed_lastcrawl', date('Y-m-d H:i:s')); $this->db->where('fed_id', $fed->fed_id); $this->db->update('feeds'); } else { $this->readerself_library->crawl_items($fed->fed_id, $sp_feed->get_items()); $lastitem = $this->db->query('SELECT itm.itm_datecreated FROM ' . $this->db->dbprefix('items') . ' AS itm WHERE itm.fed_id = ? GROUP BY itm.itm_id ORDER BY itm.itm_id DESC LIMIT 0,1', array($fed->fed_id))->row(); $parse_url = parse_url($sp_feed->get_link()); $this->db->set('fed_title', $sp_feed->get_title()); $this->db->set('fed_url', $sp_feed->get_link()); $this->db->set('fed_link', $sp_feed->subscribe_url()); if (isset($parse_url['host']) == 1) { $this->db->set('fed_host', $parse_url['host']); } if ($sp_feed->get_type() & SIMPLEPIE_TYPE_RSS_ALL) { $this->db->set('fed_type', 'rss'); } else { if ($sp_feed->get_type() & SIMPLEPIE_TYPE_ATOM_ALL) { $this->db->set('fed_type', 'atom'); } } if ($sp_feed->get_image_url()) { $this->db->set('fed_image', $sp_feed->get_image_url()); } $this->db->set('fed_description', $sp_feed->get_description()); $this->db->set('fed_lasterror', ''); $this->db->set('fed_lastcrawl', date('Y-m-d H:i:s')); if ($lastitem) { $nextcrawl = ''; //older than 96 hours, next crawl in 12 hours if ($lastitem->itm_datecreated < date('Y-m-d H:i:s', time() - 3600 * 24 * 96)) { $nextcrawl = date('Y-m-d H:i:s', time() + 3600 * 12); //older than 48 hours, next crawl in 6 hours } else { if ($lastitem->itm_datecreated < date('Y-m-d H:i:s', time() - 3600 * 48)) { $nextcrawl = date('Y-m-d H:i:s', time() + 3600 * 6); //older than 24 hours, next crawl in 3 hours } else { if ($lastitem->itm_datecreated < date('Y-m-d H:i:s', time() - 3600 * 24)) { $nextcrawl = date('Y-m-d H:i:s', time() + 3600 * 3); } } } $this->db->set('fed_nextcrawl', $nextcrawl); } $this->db->where('fed_id', $fed->fed_id); $this->db->update('feeds'); } $sp_feed->__destruct(); unset($sp_feed); } } $this->db->set('crr_time', microtime(1) - $microtime_start); if (function_exists('memory_get_peak_usage')) { $this->db->set('crr_memory', memory_get_peak_usage()); } $this->db->set('crr_feeds', $query->num_rows()); if ($errors > 0) { $this->db->set('crr_errors', $errors); } $this->db->set('crr_datecreated', date('Y-m-d H:i:s')); $this->db->insert('crawler'); if ($this->db->dbdriver == 'mysqli') { $this->db->query('OPTIMIZE TABLE categories, connections, enclosures, favorites, feeds, folders, history, items, members, share, subscriptions'); } } } $this->readerself_library->set_content($content); }
public function create() { if (!$this->axipi_session->userdata('mbr_id')) { redirect(base_url() . '?u=' . $this->input->get('u')); } $data = array(); $content = array(); $this->load->library(array('form_validation', 'analyzer_library')); if ($this->config->item('folders')) { $query = $this->db->query('SELECT flr.* FROM ' . $this->db->dbprefix('folders') . ' AS flr WHERE flr.mbr_id = ? GROUP BY flr.flr_id ORDER BY flr.flr_title ASC', array($this->member->mbr_id)); $data['folders'] = array(); $data['folders'][0] = $this->lang->line('no_folder'); if ($query->num_rows() > 0) { foreach ($query->result() as $flr) { $data['folders'][$flr->flr_id] = $flr->flr_title; } } } $this->form_validation->set_rules('url', 'lang:url_feed', 'required'); if ($this->config->item('folders')) { $this->form_validation->set_rules('folder', 'lang:folder', 'required'); } $this->form_validation->set_rules('priority', 'lang:priority', 'numeric'); //$this->form_validation->set_rules('direction', 'lang:direction', ''); $data['error'] = false; $data['feeds'] = array(); if ($this->input->post('url') && !$this->input->post('analyze_done')) { $this->analyzer_library->start($this->input->post('url')); $metas = $this->analyzer_library->metas; if (count($metas) > 0) { $data['feeds'][''] = '-'; foreach ($metas as $meta) { $add = true; $headers = get_headers($meta['href'], 1); if (isset($headers['Location']) == 1) { $meta['href'] = $headers['Location']; $headers = get_headers($meta['href'], 1); if (isset($headers['Location']) == 1) { $add = false; } } if ($add) { if ($meta['title'] == '') { $data['feeds'][$meta['href']] = $meta['href']; } else { $this->analyzer_library->encoding($meta['title']); $data['feeds'][$meta['href']] = $meta['title']; } } } } } if ($this->form_validation->run() == FALSE || count($data['feeds']) > 0) { $content = $this->load->view('subscriptions_create', $data, TRUE); } else { if ($this->config->item('folders')) { $folder = false; if ($this->input->post('folder')) { $query = $this->db->query('SELECT flr.* FROM ' . $this->db->dbprefix('folders') . ' AS flr WHERE flr.mbr_id = ? AND flr.flr_id = ? GROUP BY flr.flr_id', array($this->member->mbr_id, $this->input->post('folder'))); if ($query->num_rows() > 0) { $folder = $this->input->post('folder'); } } } $query = $this->db->query('SELECT fed.*, sub.sub_id FROM ' . $this->db->dbprefix('feeds') . ' AS fed LEFT JOIN ' . $this->db->dbprefix('subscriptions') . ' AS sub ON sub.fed_id = fed.fed_id AND sub.mbr_id = ? WHERE fed.fed_link = ? GROUP BY fed.fed_id', array($this->member->mbr_id, $this->input->post('url'))); if ($query->num_rows() == 0) { $parse_url = parse_url($this->input->post('url')); if (isset($parse_url['host']) == 1 && $parse_url['host'] == 'instagram.com' && $this->config->item('instagram/enabled')) { if ($this->config->item('instagram/access_token')) { $parts = explode('/', rtrim($parse_url['path'], '/')); $total_parts = count($parts); $last_part = $parts[$total_parts - 1]; $result = json_decode(file_get_contents('https://api.instagram.com/v1/users/search?q=' . $last_part . '&count=15&access_token=' . $this->config->item('instagram/access_token'))); if (count($result->data) == 0) { $data['error'] = 'User not found'; } else { $user_id = false; foreach ($result->data as $user) { if ($user->username == $last_part) { $user_id = $user->id; break; } } if (!$user_id) { $data['error'] = 'User not found'; } else { $result = json_decode(file_get_contents('https://api.instagram.com/v1/users/' . $user_id . '?access_token=' . $this->config->item('instagram/access_token'))); $this->db->set('fed_title', 'Instagram @' . $result->data->username); $this->db->set('fed_url', $this->input->post('url')); $this->db->set('fed_description', $result->data->bio); $this->db->set('fed_image', $result->data->profile_picture); $this->db->set('fed_link', $this->input->post('url')); if (isset($parse_url['host']) == 1) { $this->db->set('fed_host', $parse_url['host']); } $this->db->set('fed_lastcrawl', date('Y-m-d H:i:s')); $this->db->set('fed_datecreated', date('Y-m-d H:i:s')); $this->db->insert('feeds'); $fed_id = $this->db->insert_id(); $this->db->set('mbr_id', $this->member->mbr_id); $this->db->set('fed_id', $fed_id); if ($this->config->item('folders')) { if ($folder) { $this->db->set('flr_id', $folder); } } $this->db->set('sub_priority', $this->input->post('priority')); $this->db->set('sub_direction', $this->input->post('direction')); $this->db->set('sub_datecreated', date('Y-m-d H:i:s')); $this->db->insert('subscriptions'); $sub_id = $this->db->insert_id(); $result = json_decode(file_get_contents('https://api.instagram.com/v1/users/' . $user_id . '/media/recent?access_token=' . $this->config->item('instagram/access_token'))); $this->readerself_library->crawl_items_instagram($fed_id, $result->data); } } } } else { if (isset($parse_url['host']) == 1 && $parse_url['host'] == 'www.facebook.com' && $this->config->item('facebook/enabled')) { include_once 'thirdparty/facebook/autoload.php'; $fb = new Facebook\Facebook(array('app_id' => $this->config->item('facebook/id'), 'app_secret' => $this->config->item('facebook/secret'))); $fbApp = $fb->getApp(); $accessToken = $fbApp->getAccessToken(); try { $parts = explode('/', rtrim($parse_url['path'], '/')); $total_parts = count($parts); $last_part = $parts[$total_parts - 1]; $request = new Facebook\FacebookRequest($fbApp, $accessToken, 'GET', $last_part . '?fields=link,name,about'); $response = $fb->getClient()->sendRequest($request); $result = $response->getDecodedBody(); $this->db->set('fed_title', $result['name']); $this->db->set('fed_url', $result['link']); $this->db->set('fed_description', $result['about']); $this->db->set('fed_link', $result['link']); if (isset($parse_url['host']) == 1) { $this->db->set('fed_host', $parse_url['host']); } $this->db->set('fed_lastcrawl', date('Y-m-d H:i:s')); $this->db->set('fed_datecreated', date('Y-m-d H:i:s')); $this->db->insert('feeds'); $fed_id = $this->db->insert_id(); $this->db->set('mbr_id', $this->member->mbr_id); $this->db->set('fed_id', $fed_id); if ($this->config->item('folders')) { if ($folder) { $this->db->set('flr_id', $folder); } } $this->db->set('sub_priority', $this->input->post('priority')); $this->db->set('sub_direction', $this->input->post('direction')); $this->db->set('sub_datecreated', date('Y-m-d H:i:s')); $this->db->insert('subscriptions'); $sub_id = $this->db->insert_id(); $request = new Facebook\FacebookRequest($fbApp, $accessToken, 'GET', $last_part . '?fields=feed{created_time,id,message,story,full_picture,place,type,status_type,link}'); $response = $fb->getClient()->sendRequest($request); $posts = $response->getDecodedBody(); $this->readerself_library->crawl_items_facebook($fed_id, $posts['feed']['data']); redirect(base_url() . 'subscriptions/read/' . $sub_id); } catch (Facebook\Exceptions\FacebookResponseException $e) { $data['error'] = 'Graph returned an error: ' . $e->getMessage(); } catch (Facebook\Exceptions\FacebookSDKException $e) { $data['error'] = 'Facebook SDK returned an error: ' . $e->getMessage(); } } else { include_once 'thirdparty/simplepie/autoloader.php'; include_once 'thirdparty/simplepie/idn/idna_convert.class.php'; $sp_feed = new SimplePie(); $sp_feed->set_feed_url(convert_to_ascii($this->input->post('url'))); $sp_feed->enable_cache(false); $sp_feed->set_timeout(60); $sp_feed->force_feed(true); $sp_feed->init(); $sp_feed->handle_content_type(); if ($sp_feed->error()) { $data['error'] = $sp_feed->error(); } else { $parse_url = parse_url($sp_feed->get_link()); $this->db->set('fed_title', $sp_feed->get_title()); $this->db->set('fed_url', $sp_feed->get_link()); $this->db->set('fed_description', $sp_feed->get_description()); $this->db->set('fed_link', $sp_feed->subscribe_url()); if (isset($parse_url['host']) == 1) { $this->db->set('fed_host', $parse_url['host']); } $this->db->set('fed_lastcrawl', date('Y-m-d H:i:s')); $this->db->set('fed_datecreated', date('Y-m-d H:i:s')); $this->db->insert('feeds'); $fed_id = $this->db->insert_id(); $this->db->set('mbr_id', $this->member->mbr_id); $this->db->set('fed_id', $fed_id); if ($this->config->item('folders')) { if ($folder) { $this->db->set('flr_id', $folder); } } $this->db->set('sub_priority', $this->input->post('priority')); $this->db->set('sub_direction', $this->input->post('direction')); $this->db->set('sub_datecreated', date('Y-m-d H:i:s')); $this->db->insert('subscriptions'); $sub_id = $this->db->insert_id(); $data['sub_id'] = $sub_id; $data['fed_title'] = $sp_feed->get_title(); $this->readerself_library->crawl_items($fed_id, $sp_feed->get_items()); } $sp_feed->__destruct(); unset($sp_feed); } } } else { $fed = $query->row(); if (!$fed->sub_id) { $this->db->set('mbr_id', $this->member->mbr_id); $this->db->set('fed_id', $fed->fed_id); if ($this->config->item('folders')) { if ($folder) { $this->db->set('flr_id', $folder); } } $this->db->set('sub_priority', $this->input->post('priority')); $this->db->set('sub_direction', $this->input->post('direction')); $this->db->set('sub_datecreated', date('Y-m-d H:i:s')); $this->db->insert('subscriptions'); $sub_id = $this->db->insert_id(); } else { $sub_id = $fed->sub_id; } $data['sub_id'] = $sub_id; $data['fed_title'] = $fed->fed_title; } if ($data['error']) { $content = $this->load->view('subscriptions_create', $data, TRUE); } else { redirect(base_url() . 'subscriptions/read/' . $sub_id); } } $this->readerself_library->set_content($content); }
/** * Fetches all current events from this extension. * @return {Array} List of events. */ function fetch($urls = null, $initial = false) { if (!$urls) { $urls = $this->get_url(); } if (!is_array($urls)) { $urls = array($urls); } $items = array(); foreach ($urls as $url_data) { if (is_array($url_data)) { // url, key list($url, $key) = $url_data; } else { $url = $url_data; $key = ''; } $feed = new SimplePie(); $feed->enable_cache(false); $data = $this->lifestream->file_get_contents($url); $feed->set_raw_data($data); $feed->enable_order_by_date(false); $feed->force_feed(true); $success = $feed->init(); if (!$success) { $sample = substr($data, 0, 150); throw new Lifestream_FeedFetchError("Error fetching feed from {$url} ({$feed->error()})....\n\n{$sample}"); } $feed->handle_content_type(); foreach ($feed->get_items() as $row) { $rows =& $this->yield_many($row, $url, $key); foreach ($rows as $row) { if (!$row) { continue; } if (!$row['key']) { $row['key'] = $key; } if (count($row)) { $items[] = $row; } } } $feed->__destruct(); unset($feed); } return $items; }
/** * A clone of the function 'fetch_feed' in wp-includes/feed.php [line #529] * * Called from 'wprss_get_feed_items' * * @since 3.5 */ function wprss_fetch_feed($url, $source = NULL, $param_force_feed = FALSE) { // Trim the URL $url = trim($url); // Initialize the Feed $feed = new SimplePie(); // Obselete method calls ? //$feed->set_cache_class( 'WP_Feed_Cache' ); //$feed->set_file_class( 'WP_SimplePie_File' ); $feed->set_feed_url($url); $feed->set_autodiscovery_level(SIMPLEPIE_LOCATOR_ALL); // If a feed source was passed if ($source !== NULL || $param_force_feed) { // Get the force feed option for the feed source $force_feed = get_post_meta($source, 'wprss_force_feed', TRUE); // If turned on, force the feed if ($force_feed == 'true' || $param_force_feed) { $feed->force_feed(TRUE); } } // Set timeout limit $fetch_time_limit = wprss_get_feed_fetch_time_limit(); $feed->set_timeout($fetch_time_limit); //$feed->set_cache_duration( apply_filters( 'wp_feed_cache_transient_lifetime', 12 * HOUR_IN_SECONDS, $url ) ); $feed->enable_cache(FALSE); // Reference array action hook, for the feed object and the URL do_action_ref_array('wp_feed_options', array(&$feed, $url)); // Prepare the tags to strip from the feed $tags_to_strip = apply_filters('wprss_feed_tags_to_strip', $feed->strip_htmltags, $source); // Strip them $feed->strip_htmltags($tags_to_strip); do_action('wprss_fetch_feed_before', $feed); // Fetch the feed $feed->init(); $feed->handle_content_type(); do_action('wprss_fetch_feed_after', $feed); // Convert the feed error into a WP_Error, if applicable if ($feed->error()) { if ($source !== NULL) { $msg = sprintf(__('Failed to fetch the RSS feed. Error: %s', WPRSS_TEXT_DOMAIN), $feed->error()); update_post_meta($source, 'wprss_error_last_import', $msg); } return new WP_Error('simplepie-error', $feed->error()); } // If no error, return the feed and remove any error meta delete_post_meta($source, "wprss_error_last_import"); return $feed; }
/** * Renders a feed * * @param string $url - the feed url * * @return void */ public static function renderFeed($url) { $rssitems = 5; $rssitemdesc = 1; // Aaach, Joomla 2.5 please die faster... if (JVERSION < '3') { jimport('simplepie.simplepie'); $cache = JFactory::getCache('feed_parser', 'callback'); $cache->setLifeTime(600); $simplepie = new SimplePie(null, null, 0); $simplepie->enable_cache(false); $simplepie->set_feed_url($url); $simplepie->force_feed(true); $contents = $cache->get(array($simplepie, 'init'), null, false, false); if ($contents) { $rssDoc = $simplepie; } else { return JText::_('LIB_COMPOJOOM_FEED_COULDNT_BE_FETCHED'); } } else { // Get RSS parsed object try { jimport('joomla.feed.factory'); $feed = new JFeedFactory(); $rssDoc = $feed->getFeed($url); } catch (Exception $e) { return JText::_('LIB_COMPOJOOM_FEED_COULDNT_BE_FETCHED'); } } $feed = $rssDoc; if (JVERSION < 3) { if ($rssDoc != false) { $filter = JFilterInput::getInstance(); // Channel header and link $channel['title'] = $filter->clean($rssDoc->get_title()); $channel['link'] = $filter->clean($rssDoc->get_link()); $channel['description'] = $filter->clean($rssDoc->get_description()); // Items $items = $rssDoc->get_items(); // Feed elements $items = array_slice($items, 0, $rssitems); ?> <div class="newsfeed"> <?php if (!is_null($channel['title'])) { ?> <h2> <a href="<?php echo htmlspecialchars(str_replace('&', '&', $channel['link'])); ?> " target="_blank"> <?php echo htmlspecialchars($channel['title']); ?> </a> </h2> <?php } ?> <?php echo $channel['description']; ?> <?php $actualItems = count($items); $setItems = $rssitems; if ($setItems > $actualItems) { $totalItems = $actualItems; } else { $totalItems = $setItems; } ?> <ul class="newsfeed"> <?php for ($j = 0; $j < $totalItems; $j++) { $currItem = $items[$j]; ?> <li> <?php if (!is_null($currItem->get_link())) { ?> <a href="<?php echo htmlspecialchars($currItem->get_link()); ?> " target="_child"> <?php echo htmlspecialchars($currItem->get_title()); ?> </a> <?php } ?> <?php // Item description if ($rssitemdesc) { // Item description $text = $filter->clean(html_entity_decode($currItem->get_description(), ENT_COMPAT, 'UTF-8')); $text = str_replace(''', "'", $text); ?> <div> <?php echo CompojoomHtmlString::truncateComplex($text, 200); ?> </div> <?php } ?> </li> <?php } ?> </ul> </div> <?php } } else { if ($rssDoc != false) { ?> <div class="feed"> <?php if (!is_null($feed->title)) { ?> <h2> <a href="<?php echo str_replace('&', '&', $url); ?> " target="_blank"> <?php echo $feed->title; ?> </a> </h2> <?php } ?> <?php echo $feed->description; ?> <ul class="newsfeed"> <?php for ($i = 0; $i < $rssitems; $i++) { if (!$feed->offsetExists($i)) { break; } ?> <?php $uri = !empty($feed[$i]->uri) || !is_null($feed[$i]->uri) ? $feed[$i]->uri : $feed[$i]->guid; $text = !empty($feed[$i]->content) || !is_null($feed[$i]->content) ? $feed[$i]->content : $feed[$i]->description; ?> <li> <?php if (!empty($uri)) { ?> <h5 class="feed-link"> <a href="<?php echo $uri; ?> " target="_blank"> <?php echo $feed[$i]->title; ?> </a></h5> <?php } else { ?> <h5 class="feed-link"><?php echo $feed[$i]->title; ?> </h5> <?php } ?> <?php if ($rssitemdesc && !empty($text)) { ?> <div class="feed-item-description"> <?php // Strip the images. $text = JFilterOutput::stripImages($text); $text = CompojoomHtmlString::truncateComplex($text, 200); echo str_replace(''', "'", $text); ?> </div> <?php } ?> </li> <?php } ?> </ul> </div> <?php } } }
public static function get_items_feed($url, $params = null) { $feed = new SimplePie(); $mode = isset($params->mode) ? $params->mode : 0; if ($mode == 0) { $feed->set_feed_url($url); } else { $html = ogbFile::get_curl($url); $feed->set_raw_data($html); } $feed->set_autodiscovery_level(SIMPLEPIE_LOCATOR_NONE); $feed->set_timeout(20); $feed->enable_cache(false); $feed->set_stupidly_fast(true); $feed->enable_order_by_date(false); // we don't want to do anything to the feed $feed->set_url_replacements(array()); $feed->force_feed(true); $result = $feed->init(); if (isset($_GET['x'])) { echo "\n\n<br /><i><b>File:</b>" . __FILE__ . ' <b>Line:</b>' . __LINE__ . "</i><br />\n\n"; echo "<p>URL: [{$url}]</p>"; echo "<p>Error: [{$feed->error}]</p>"; } $items = $feed->get_items(); $c_items = count($items); if ($c_items == 0) { echo "<p>Error: [{$feed->error}]</p>"; return array(); } for ($i = 0; $i < count($items); $i++) { $row = new stdclass(); $row->title = $items[$i]->get_title(); # the title for the post $row->link = $items[$i]->get_link(); # a single link for the post $row->description = $items[$i]->get_description(); # the content of the post (prefers summaries) $row->author = $items[$i]->get_author(); # a single author for the post $row->date = $items[$i]->get_date('Y-m-d H:i:s'); $row->enclosures = $items[$i]->get_enclosures(); $rows[] = $row; } return $rows; }
/** * Get a specific RSS feed * * @param $url string/array URL of the feed or array of URL * @param $cache_duration timestamp cache duration (default DAY_TIMESTAMP) * * @return feed object **/ static function getRSSFeed($url, $cache_duration = DAY_TIMESTAMP) { $feed = new SimplePie(); $feed->set_cache_location(GLPI_RSS_DIR); $feed->set_cache_duration($cache_duration); $feed->enable_cache(true); $feed->set_feed_url($url); $feed->force_feed(true); // Initialize the whole SimplePie object. Read the feed, process it, parse it, cache it, and // all that other good stuff. The feed's information will not be available to SimplePie before // this is called. $feed->init(); // We'll make sure that the right content type and character encoding gets set automatically. // This function will grab the proper character encoding, as well as set the content type to text/html. $feed->handle_content_type(); if ($feed->error()) { return false; } return $feed; }
/** * Get a parsed XML Feed Source * * @param string $url Url for feed source. * @param integer $cache_time Time to cache feed for (using internal cache mechanism). * * @return mixed SimplePie parsed object on success, false on failure. * * @since 11.1 */ public static function getFeedParser($url, $cache_time = 0) { jimport('simplepie.simplepie'); $cache = self::getCache('feed_parser', 'callback'); if ($cache_time > 0) { $cache->setLifeTime($cache_time); } $simplepie = new SimplePie(null, null, 0); $simplepie->enable_cache(false); $simplepie->set_feed_url($url); $simplepie->force_feed(true); $contents = $cache->get(array($simplepie, 'init'), null, false, false); if ($contents) { return $simplepie; } else { JLog::add(JText::_('JLIB_UTIL_ERROR_LOADING_FEED_DATA'), JLog::WARNING, 'jerror'); } return false; }
private function GetRssItems($isPreview, $text) { // Make sure we have the cache location configured $file = new File($this->db); File::EnsureLibraryExists(); // Make sure we have a $media/$layout object to use $media = new Media(); $layout = new Layout(); // Parse the text template $matches = ''; preg_match_all('/\\[.*?\\]/', $text, $matches); Debug::LogEntry('audit', 'Loading SimplePie to handle RSS parsing.' . urldecode($this->GetOption('uri'))); // Use SimplePie to get the feed include_once '3rdparty/simplepie/autoloader.php'; $feed = new SimplePie(); $feed->set_cache_location($file->GetLibraryCacheUri()); $feed->set_feed_url(urldecode($this->GetOption('uri'))); $feed->force_feed(true); $feed->set_cache_duration($this->GetOption('updateInterval', 3600) * 60); $feed->handle_content_type(); // Get a list of allowed attributes if ($this->GetOption('allowedAttributes') != '') { $attrsStrip = array_diff($feed->strip_attributes, explode(',', $this->GetOption('allowedAttributes'))); //Debug::Audit(var_export($attrsStrip, true)); $feed->strip_attributes($attrsStrip); } // Disable date sorting? if ($this->GetOption('disableDateSort') == 1) { $feed->enable_order_by_date(false); } // Init $feed->init(); $dateFormat = $this->GetOption('dateFormat'); if ($feed->error()) { Debug::LogEntry('audit', 'Feed Error: ' . $feed->error()); return array(); } // Set an expiry time for the media $expires = time() + $this->GetOption('updateInterval', 3600) * 60; // Store our formatted items $items = array(); foreach ($feed->get_items() as $item) { /* @var SimplePie_Item $item */ // Substitute for all matches in the template $rowString = $text; // Substitute foreach ($matches[0] as $sub) { $replace = ''; // Pick the appropriate column out if (strstr($sub, '|') !== false) { // Use the provided name space to extract a tag $attribs = NULL; if (substr_count($sub, '|') > 1) { list($tag, $namespace, $attribs) = explode('|', $sub); } else { list($tag, $namespace) = explode('|', $sub); } // What are we looking at Debug::Audit('Namespace: ' . str_replace(']', '', $namespace) . '. Tag: ' . str_replace('[', '', $tag) . '. '); // Are we an image place holder? if (strstr($namespace, 'image') != false) { // Try to get a link for the image $link = null; switch (str_replace('[', '', $tag)) { case 'Link': if ($enclosure = $item->get_enclosure()) { // Use the link to get the image $link = $enclosure->get_link(); } break; default: // Default behaviour just tries to get the content from the tag provided (without a name space). $tags = $item->get_item_tags('', str_replace('[', '', $tag)); if ($tags != null) { $link = is_array($tags) ? $tags[0]['data'] : ''; } } if ($link == NULL) { $dom = new DOMDocument(); $dom->loadHTML($item->get_content()); // Full $images = $dom->getElementsByTagName('img'); foreach ($images as $key => $value) { if ($key == 0) { $link = html_entity_decode($images->item($key)->getAttribute('src')); } } } if ($link == NULL) { $dom = new DOMDocument(); $dom->loadHTML($item->get_description()); //Summary $images = $dom->getElementsByTagName('img'); foreach ($images as $key => $value) { if ($key == 0) { $link = html_entity_decode($images->item($key)->getAttribute('src')); } } } // If we have managed to resolve a link, download it and replace the tag with the downloaded // image url if ($link != NULL) { // Grab the profile image $file = $media->addModuleFileFromUrl($link, 'ticker_' . md5($this->GetOption('url') . $link), $expires); // Tag this layout with this file $layout->AddLk($this->layoutid, 'module', $file['mediaId']); $replace = $isPreview ? '<img src="index.php?p=module&mod=image&q=Exec&method=GetResource&mediaid=' . $file['mediaId'] . '" ' . $attribs . '/>' : '<img src="' . $file['storedAs'] . '" ' . $attribs . ' />'; } } else { $tags = $item->get_item_tags(str_replace(']', '', $namespace), str_replace('[', '', $tag)); Debug::LogEntry('audit', 'Tags:' . var_export($tags, true)); // If we find some tags then do the business with them if ($tags != NULL) { if ($attribs != NULL) { $replace = is_array($tags) ? $tags[0]['attribs'][''][str_replace(']', '', $attribs)] : ''; } else { $replace = is_array($tags) ? $tags[0]['data'] : ''; } } } } else { // Use the pool of standard tags switch ($sub) { case '[Name]': $replace = $this->GetOption('name'); break; case '[Title]': $replace = $item->get_title(); break; case '[Description]': $replace = $item->get_description(); break; case '[Content]': $replace = $item->get_content(); break; case '[Copyright]': $replace = $item->get_copyright(); break; case '[Date]': $replace = DateManager::getLocalDate($item->get_date('U'), $dateFormat); break; case '[PermaLink]': $replace = $item->get_permalink(); break; case '[Link]': $replace = $item->get_link(); break; } if ($this->GetOption('stripTags') != '') { require_once '3rdparty/htmlpurifier/library/HTMLPurifier.auto.php'; $config = HTMLPurifier_Config::createDefault(); $config->set('HTML.ForbiddenElements', array_merge($feed->strip_htmltags, explode(',', $this->GetOption('stripTags')))); $purifier = new HTMLPurifier($config); $replace = $purifier->purify($replace); } } // Substitute the replacement we have found (it might be '') $rowString = str_replace($sub, $replace, $rowString); } $items[] = $rowString; } // Copyright information? if ($this->GetOption('copyright', '') != '') { $items[] = '<span id="copyright">' . $this->GetOption('copyright') . '</span>'; } // Return the formatted items return $items; }
function update_rss_feed_real($link, $feed, $ignore_daemon = false, $no_cache = false, $override_url = false) { global $memcache; $debug_enabled = defined('DAEMON_EXTENDED_DEBUG') || $_REQUEST['xdebug']; if (!$_REQUEST["daemon"] && !$ignore_daemon) { return false; } if ($debug_enabled) { _debug("update_rss_feed: start"); } if (!$ignore_daemon) { if (DB_TYPE == "pgsql") { $updstart_thresh_qpart = "(ttrss_feeds.last_update_started IS NULL OR ttrss_feeds.last_update_started < NOW() - INTERVAL '120 seconds')"; } else { $updstart_thresh_qpart = "(ttrss_feeds.last_update_started IS NULL OR ttrss_feeds.last_update_started < DATE_SUB(NOW(), INTERVAL 120 SECOND))"; } $result = db_query($link, "SELECT id,update_interval,auth_login,\n\t\t\t\tauth_pass,cache_images,update_method\n\t\t\t\tFROM ttrss_feeds WHERE id = '{$feed}' AND {$updstart_thresh_qpart}"); } else { $result = db_query($link, "SELECT id,update_interval,auth_login,\n\t\t\t\tfeed_url,auth_pass,cache_images,update_method,last_updated,\n\t\t\t\tmark_unread_on_update, owner_uid, update_on_checksum_change,\n\t\t\t\tpubsub_state\n\t\t\t\tFROM ttrss_feeds WHERE id = '{$feed}'"); } if (db_num_rows($result) == 0) { if ($debug_enabled) { _debug("update_rss_feed: feed {$feed} NOT FOUND/SKIPPED"); } return false; } $update_method = db_fetch_result($result, 0, "update_method"); $last_updated = db_fetch_result($result, 0, "last_updated"); $owner_uid = db_fetch_result($result, 0, "owner_uid"); $mark_unread_on_update = sql_bool_to_bool(db_fetch_result($result, 0, "mark_unread_on_update")); $update_on_checksum_change = sql_bool_to_bool(db_fetch_result($result, 0, "update_on_checksum_change")); $pubsub_state = db_fetch_result($result, 0, "pubsub_state"); db_query($link, "UPDATE ttrss_feeds SET last_update_started = NOW()\n\t\t\tWHERE id = '{$feed}'"); $auth_login = db_fetch_result($result, 0, "auth_login"); $auth_pass = db_fetch_result($result, 0, "auth_pass"); if ($update_method == 0) { $update_method = DEFAULT_UPDATE_METHOD + 1; } // 1 - Magpie // 2 - SimplePie // 3 - Twitter OAuth if ($update_method == 2) { $use_simplepie = true; } else { $use_simplepie = false; } if ($debug_enabled) { _debug("update method: {$update_method} (feed setting: {$update_method}) (use simplepie: {$use_simplepie})\n"); } if ($update_method == 1) { $auth_login = urlencode($auth_login); $auth_pass = urlencode($auth_pass); } $update_interval = db_fetch_result($result, 0, "update_interval"); $cache_images = sql_bool_to_bool(db_fetch_result($result, 0, "cache_images")); $fetch_url = db_fetch_result($result, 0, "feed_url"); if ($update_interval < 0) { return false; } $feed = db_escape_string($feed); if ($auth_login && $auth_pass) { $url_parts = array(); preg_match("/(^[^:]*):\\/\\/(.*)/", $fetch_url, $url_parts); if ($url_parts[1] && $url_parts[2]) { $fetch_url = $url_parts[1] . "://{$auth_login}:{$auth_pass}@" . $url_parts[2]; } } if ($override_url) { $fetch_url = $override_url; } if ($debug_enabled) { _debug("update_rss_feed: fetching [{$fetch_url}]..."); } $obj_id = md5("FDATA:{$use_simplepie}:{$fetch_url}"); if ($memcache && ($obj = $memcache->get($obj_id))) { if ($debug_enabled) { _debug("update_rss_feed: data found in memcache."); } $rss = $obj; } else { if ($update_method == 3) { $rss = fetch_twitter_rss($link, $fetch_url, $owner_uid); } else { if ($update_method == 1) { define('MAGPIE_CACHE_AGE', get_feed_update_interval($link, $feed) * 60); define('MAGPIE_CACHE_ON', !$no_cache); define('MAGPIE_FETCH_TIME_OUT', 60); define('MAGPIE_CACHE_DIR', CACHE_DIR . "/magpie"); $rss = @fetch_rss($fetch_url); } else { $simplepie_cache_dir = CACHE_DIR . "/simplepie"; if (!is_dir($simplepie_cache_dir)) { mkdir($simplepie_cache_dir); } $rss = new SimplePie(); $rss->set_useragent(SELF_USER_AGENT); # $rss->set_timeout(10); $rss->set_feed_url($fetch_url); $rss->set_output_encoding('UTF-8'); $rss->force_feed(true); if (SIMPLEPIE_CACHE_IMAGES && $cache_images) { if ($debug_enabled) { _debug("enabling image cache"); } $rss->set_image_handler("image.php", 'i'); } if ($debug_enabled) { _debug("feed update interval (sec): " . get_feed_update_interval($link, $feed) * 60); } $rss->enable_cache(!$no_cache); if (!$no_cache) { $rss->set_cache_location($simplepie_cache_dir); $rss->set_cache_duration(get_feed_update_interval($link, $feed) * 60); } $rss->init(); } } if ($memcache && $rss) { $memcache->add($obj_id, $rss, 0, 300); } } // print_r($rss); if ($debug_enabled) { _debug("update_rss_feed: fetch done, parsing..."); } $feed = db_escape_string($feed); if ($update_method == 2) { $fetch_ok = !$rss->error(); } else { $fetch_ok = !!$rss; } if ($fetch_ok) { if ($debug_enabled) { _debug("update_rss_feed: processing feed data..."); } // db_query($link, "BEGIN"); $result = db_query($link, "SELECT title,icon_url,site_url,owner_uid\n\t\t\t\tFROM ttrss_feeds WHERE id = '{$feed}'"); $registered_title = db_fetch_result($result, 0, "title"); $orig_icon_url = db_fetch_result($result, 0, "icon_url"); $orig_site_url = db_fetch_result($result, 0, "site_url"); $owner_uid = db_fetch_result($result, 0, "owner_uid"); if ($use_simplepie) { $site_url = $rss->get_link(); } else { $site_url = $rss->channel["link"]; } $site_url = rewrite_relative_url($fetch_url, $site_url); if ($debug_enabled) { _debug("update_rss_feed: checking favicon..."); } check_feed_favicon($site_url, $feed, $link); if (!$registered_title || $registered_title == "[Unknown]") { if ($use_simplepie) { $feed_title = db_escape_string($rss->get_title()); } else { $feed_title = db_escape_string($rss->channel["title"]); } if ($debug_enabled) { _debug("update_rss_feed: registering title: {$feed_title}"); } db_query($link, "UPDATE ttrss_feeds SET\n\t\t\t\t\ttitle = '{$feed_title}' WHERE id = '{$feed}'"); } // weird, weird Magpie if (!$use_simplepie) { if (!$site_url) { $site_url = db_escape_string($rss->channel["link_"]); } } if ($site_url && $orig_site_url != db_escape_string($site_url)) { db_query($link, "UPDATE ttrss_feeds SET\n\t\t\t\t\tsite_url = '{$site_url}' WHERE id = '{$feed}'"); } // print "I: " . $rss->channel["image"]["url"]; if (!$use_simplepie) { $icon_url = db_escape_string($rss->image["url"]); } else { $icon_url = db_escape_string($rss->get_image_url()); } $icon_url = substr($icon_url, 0, 250); if ($icon_url && $orig_icon_url != $icon_url) { db_query($link, "UPDATE ttrss_feeds SET icon_url = '{$icon_url}' WHERE id = '{$feed}'"); } if ($debug_enabled) { _debug("update_rss_feed: loading filters..."); } $filters = load_filters($link, $feed, $owner_uid); // if ($debug_enabled) { // print_r($filters); // } if ($use_simplepie) { $iterator = $rss->get_items(); } else { $iterator = $rss->items; if (!$iterator || !is_array($iterator)) { $iterator = $rss->entries; } if (!$iterator || !is_array($iterator)) { $iterator = $rss; } } if (!is_array($iterator)) { /* db_query($link, "UPDATE ttrss_feeds SET last_error = 'Parse error: can\'t find any articles.' WHERE id = '$feed'"); */ // clear any errors and mark feed as updated if fetched okay // even if it's blank if ($debug_enabled) { _debug("update_rss_feed: entry iterator is not an array, no articles?"); } db_query($link, "UPDATE ttrss_feeds\n\t\t\t\t\tSET last_updated = NOW(), last_error = '' WHERE id = '{$feed}'"); return; // no articles } if ($pubsub_state != 2 && PUBSUBHUBBUB_ENABLED) { if ($debug_enabled) { _debug("update_rss_feed: checking for PUSH hub..."); } $feed_hub_url = false; if ($use_simplepie) { $links = $rss->get_links('hub'); if ($links && is_array($links)) { foreach ($links as $l) { $feed_hub_url = $l; break; } } } else { $atom = $rss->channel['atom']; if ($atom) { if ($atom['link@rel'] == 'hub') { $feed_hub_url = $atom['link@href']; } if (!$feed_hub_url && $atom['link#'] > 1) { for ($i = 2; $i <= $atom['link#']; $i++) { if ($atom["link#{$i}@rel"] == 'hub') { $feed_hub_url = $atom["link#{$i}@href"]; break; } } } } else { $feed_hub_url = $rss->channel['link_hub']; } } if ($debug_enabled) { _debug("update_rss_feed: feed hub url: {$feed_hub_url}"); } if ($feed_hub_url && function_exists('curl_init') && !ini_get("open_basedir")) { $callback_url = get_self_url_prefix() . "/public.php?op=pubsub&id={$feed}"; $s = new Subscriber($feed_hub_url, $callback_url); $rc = $s->subscribe($fetch_url); if ($debug_enabled) { _debug("update_rss_feed: feed hub url found, subscribe request sent."); } db_query($link, "UPDATE ttrss_feeds SET pubsub_state = 1\n\t\t\t\t\t\tWHERE id = '{$feed}'"); } } if ($debug_enabled) { _debug("update_rss_feed: processing articles..."); } foreach ($iterator as $item) { if ($_REQUEST['xdebug'] == 2) { print_r($item); } if ($use_simplepie) { $entry_guid = $item->get_id(); if (!$entry_guid) { $entry_guid = $item->get_link(); } if (!$entry_guid) { $entry_guid = make_guid_from_title($item->get_title()); } } else { $entry_guid = $item["id"]; if (!$entry_guid) { $entry_guid = $item["guid"]; } if (!$entry_guid) { $entry_guid = $item["about"]; } if (!$entry_guid) { $entry_guid = $item["link"]; } if (!$entry_guid) { $entry_guid = make_guid_from_title($item["title"]); } } if ($debug_enabled) { _debug("update_rss_feed: guid {$entry_guid}"); } if (!$entry_guid) { continue; } $entry_timestamp = ""; if ($use_simplepie) { $entry_timestamp = strtotime($item->get_date()); } else { $rss_2_date = $item['pubdate']; $rss_1_date = $item['dc']['date']; $atom_date = $item['issued']; if (!$atom_date) { $atom_date = $item['updated']; } if ($atom_date != "") { $entry_timestamp = parse_w3cdtf($atom_date); } if ($rss_1_date != "") { $entry_timestamp = parse_w3cdtf($rss_1_date); } if ($rss_2_date != "") { $entry_timestamp = strtotime($rss_2_date); } } if ($entry_timestamp == "" || $entry_timestamp == -1 || !$entry_timestamp) { $entry_timestamp = time(); $no_orig_date = 'true'; } else { $no_orig_date = 'false'; } $entry_timestamp_fmt = strftime("%Y/%m/%d %H:%M:%S", $entry_timestamp); if ($debug_enabled) { _debug("update_rss_feed: date {$entry_timestamp} [{$entry_timestamp_fmt}]"); } if ($use_simplepie) { $entry_title = $item->get_title(); } else { $entry_title = trim(strip_tags($item["title"])); } if ($use_simplepie) { $entry_link = $item->get_link(); } else { // strange Magpie workaround $entry_link = $item["link_"]; if (!$entry_link) { $entry_link = $item["link"]; } } $entry_link = rewrite_relative_url($site_url, $entry_link); if ($debug_enabled) { _debug("update_rss_feed: title {$entry_title}"); _debug("update_rss_feed: link {$entry_link}"); } if (!$entry_title) { $entry_title = date("Y-m-d H:i:s", $entry_timestamp); } $entry_link = strip_tags($entry_link); if ($use_simplepie) { $entry_content = $item->get_content(); if (!$entry_content) { $entry_content = $item->get_description(); } } else { $entry_content = $item["content:escaped"]; if (!$entry_content) { $entry_content = $item["content:encoded"]; } if (!$entry_content) { $entry_content = $item["content"]["encoded"]; } if (!$entry_content) { $entry_content = $item["content"]; } // Magpie bugs are getting ridiculous if (trim($entry_content) == "Array") { $entry_content = false; } if (!$entry_content) { $entry_content = $item["atom_content"]; } if (!$entry_content) { $entry_content = $item["summary"]; } if (!$entry_content || strlen($entry_content) < strlen($item["description"])) { $entry_content = $item["description"]; } // WTF if (is_array($entry_content)) { $entry_content = $entry_content["encoded"]; if (!$entry_content) { $entry_content = $entry_content["escaped"]; } } } if ($_REQUEST["xdebug"] == 2) { print "update_rss_feed: content: "; print_r(htmlspecialchars($entry_content)); } $entry_content_unescaped = $entry_content; if ($use_simplepie) { $entry_comments = strip_tags($item->data["comments"]); if ($item->get_author()) { $entry_author_item = $item->get_author(); $entry_author = $entry_author_item->get_name(); if (!$entry_author) { $entry_author = $entry_author_item->get_email(); } $entry_author = db_escape_string($entry_author); } } else { $entry_comments = strip_tags($item["comments"]); $entry_author = db_escape_string(strip_tags($item['dc']['creator'])); if ($item['author']) { if (is_array($item['author'])) { if (!$entry_author) { $entry_author = db_escape_string(strip_tags($item['author']['name'])); } if (!$entry_author) { $entry_author = db_escape_string(strip_tags($item['author']['email'])); } } if (!$entry_author) { $entry_author = db_escape_string(strip_tags($item['author'])); } } } if (preg_match('/^[\\t\\n\\r ]*$/', $entry_author)) { $entry_author = ''; } $entry_guid = db_escape_string(strip_tags($entry_guid)); $entry_guid = mb_substr($entry_guid, 0, 250); $result = db_query($link, "SELECT id FROM\tttrss_entries\n\t\t\t\t\tWHERE guid = '{$entry_guid}'"); $entry_content = db_escape_string($entry_content, false); $content_hash = "SHA1:" . sha1(strip_tags($entry_content)); $entry_title = db_escape_string($entry_title); $entry_link = db_escape_string($entry_link); $entry_comments = mb_substr(db_escape_string($entry_comments), 0, 250); $entry_author = mb_substr($entry_author, 0, 250); if ($use_simplepie) { $num_comments = 0; #FIXME# } else { $num_comments = db_escape_string($item["slash"]["comments"]); } if (!$num_comments) { $num_comments = 0; } if ($debug_enabled) { _debug("update_rss_feed: looking for tags [1]..."); } // parse <category> entries into tags $additional_tags = array(); if ($use_simplepie) { $additional_tags_src = $item->get_categories(); if (is_array($additional_tags_src)) { foreach ($additional_tags_src as $tobj) { array_push($additional_tags, $tobj->get_term()); } } if ($debug_enabled) { _debug("update_rss_feed: category tags:"); print_r($additional_tags); } } else { $t_ctr = $item['category#']; if ($t_ctr == 0) { $additional_tags = array(); } else { if ($t_ctr > 0) { $additional_tags = array($item['category']); if ($item['category@term']) { array_push($additional_tags, $item['category@term']); } for ($i = 0; $i <= $t_ctr; $i++) { if ($item["category#{$i}"]) { array_push($additional_tags, $item["category#{$i}"]); } if ($item["category#{$i}@term"]) { array_push($additional_tags, $item["category#{$i}@term"]); } } } } // parse <dc:subject> elements $t_ctr = $item['dc']['subject#']; if ($t_ctr > 0) { array_push($additional_tags, $item['dc']['subject']); for ($i = 0; $i <= $t_ctr; $i++) { if ($item['dc']["subject#{$i}"]) { array_push($additional_tags, $item['dc']["subject#{$i}"]); } } } } if ($debug_enabled) { _debug("update_rss_feed: looking for tags [2]..."); } /* taaaags */ // <a href="..." rel="tag">Xorg</a>, // $entry_tags = null; preg_match_all("/<a.*?rel=['\"]tag['\"].*?\\>([^<]+)<\\/a>/i", $entry_content_unescaped, $entry_tags); $entry_tags = $entry_tags[1]; $entry_tags = array_merge($entry_tags, $additional_tags); $entry_tags = array_unique($entry_tags); for ($i = 0; $i < count($entry_tags); $i++) { $entry_tags[$i] = mb_strtolower($entry_tags[$i], 'utf-8'); } if ($debug_enabled) { _debug("update_rss_feed: unfiltered tags found:"); print_r($entry_tags); } # sanitize content $entry_content = sanitize_article_content($entry_content); $entry_title = sanitize_article_content($entry_title); if ($debug_enabled) { _debug("update_rss_feed: done collecting data [TITLE:{$entry_title}]"); } db_query($link, "BEGIN"); if (db_num_rows($result) == 0) { if ($debug_enabled) { _debug("update_rss_feed: base guid not found"); } // base post entry does not exist, create it $result = db_query($link, "INSERT INTO ttrss_entries\n\t\t\t\t\t\t\t(title,\n\t\t\t\t\t\t\tguid,\n\t\t\t\t\t\t\tlink,\n\t\t\t\t\t\t\tupdated,\n\t\t\t\t\t\t\tcontent,\n\t\t\t\t\t\t\tcontent_hash,\n\t\t\t\t\t\t\tno_orig_date,\n\t\t\t\t\t\t\tdate_updated,\n\t\t\t\t\t\t\tdate_entered,\n\t\t\t\t\t\t\tcomments,\n\t\t\t\t\t\t\tnum_comments,\n\t\t\t\t\t\t\tauthor)\n\t\t\t\t\t\tVALUES\n\t\t\t\t\t\t\t('{$entry_title}',\n\t\t\t\t\t\t\t'{$entry_guid}',\n\t\t\t\t\t\t\t'{$entry_link}',\n\t\t\t\t\t\t\t'{$entry_timestamp_fmt}',\n\t\t\t\t\t\t\t'{$entry_content}',\n\t\t\t\t\t\t\t'{$content_hash}',\n\t\t\t\t\t\t\t{$no_orig_date},\n\t\t\t\t\t\t\tNOW(),\n\t\t\t\t\t\t\tNOW(),\n\t\t\t\t\t\t\t'{$entry_comments}',\n\t\t\t\t\t\t\t'{$num_comments}',\n\t\t\t\t\t\t\t'{$entry_author}')"); } else { // we keep encountering the entry in feeds, so we need to // update date_updated column so that we don't get horrible // dupes when the entry gets purged and reinserted again e.g. // in the case of SLOW SLOW OMG SLOW updating feeds $base_entry_id = db_fetch_result($result, 0, "id"); db_query($link, "UPDATE ttrss_entries SET date_updated = NOW()\n\t\t\t\t\t\tWHERE id = '{$base_entry_id}'"); } // now it should exist, if not - bad luck then $result = db_query($link, "SELECT\n\t\t\t\t\t\tid,content_hash,no_orig_date,title,\n\t\t\t\t\t\t" . SUBSTRING_FOR_DATE . "(date_updated,1,19) as date_updated,\n\t\t\t\t\t\t" . SUBSTRING_FOR_DATE . "(updated,1,19) as updated,\n\t\t\t\t\t\tnum_comments\n\t\t\t\t\tFROM\n\t\t\t\t\t\tttrss_entries\n\t\t\t\t\tWHERE guid = '{$entry_guid}'"); $entry_ref_id = 0; $entry_int_id = 0; if (db_num_rows($result) == 1) { if ($debug_enabled) { _debug("update_rss_feed: base guid found, checking for user record"); } // this will be used below in update handler $orig_content_hash = db_fetch_result($result, 0, "content_hash"); $orig_title = db_fetch_result($result, 0, "title"); $orig_num_comments = db_fetch_result($result, 0, "num_comments"); $orig_date_updated = strtotime(db_fetch_result($result, 0, "date_updated")); $ref_id = db_fetch_result($result, 0, "id"); $entry_ref_id = $ref_id; // check for user post link to main table // do we allow duplicate posts with same GUID in different feeds? if (get_pref($link, "ALLOW_DUPLICATE_POSTS", $owner_uid, false)) { $dupcheck_qpart = "AND (feed_id = '{$feed}' OR feed_id IS NULL)"; } else { $dupcheck_qpart = ""; } /* Collect article tags here so we could filter by them: */ $article_filters = get_article_filters($filters, $entry_title, $entry_content, $entry_link, $entry_timestamp, $entry_author, $entry_tags); if ($debug_enabled) { _debug("update_rss_feed: article filters: "); if (count($article_filters) != 0) { print_r($article_filters); } } if (find_article_filter($article_filters, "filter")) { db_query($link, "COMMIT"); // close transaction in progress continue; } $score = calculate_article_score($article_filters); if ($debug_enabled) { _debug("update_rss_feed: initial score: {$score}"); } $query = "SELECT ref_id, int_id FROM ttrss_user_entries WHERE\n\t\t\t\t\t\t\tref_id = '{$ref_id}' AND owner_uid = '{$owner_uid}'\n\t\t\t\t\t\t\t{$dupcheck_qpart}"; // if ($_REQUEST["xdebug"]) print "$query\n"; $result = db_query($link, $query); // okay it doesn't exist - create user entry if (db_num_rows($result) == 0) { if ($debug_enabled) { _debug("update_rss_feed: user record not found, creating..."); } if ($score >= -500 && !find_article_filter($article_filters, 'catchup')) { $unread = 'true'; $last_read_qpart = 'NULL'; } else { $unread = 'false'; $last_read_qpart = 'NOW()'; } if (find_article_filter($article_filters, 'mark') || $score > 1000) { $marked = 'true'; } else { $marked = 'false'; } if (find_article_filter($article_filters, 'publish')) { $published = 'true'; } else { $published = 'false'; } $result = db_query($link, "INSERT INTO ttrss_user_entries\n\t\t\t\t\t\t\t\t(ref_id, owner_uid, feed_id, unread, last_read, marked,\n\t\t\t\t\t\t\t\t\tpublished, score, tag_cache, label_cache, uuid)\n\t\t\t\t\t\t\tVALUES ('{$ref_id}', '{$owner_uid}', '{$feed}', {$unread},\n\t\t\t\t\t\t\t\t{$last_read_qpart}, {$marked}, {$published}, '{$score}', '', '', '')"); if (PUBSUBHUBBUB_HUB && $published == 'true') { $rss_link = get_self_url_prefix() . "/public.php?op=rss&id=-2&key=" . get_feed_access_key($link, -2, false, $owner_uid); $p = new Publisher(PUBSUBHUBBUB_HUB); $pubsub_result = $p->publish_update($rss_link); } $result = db_query($link, "SELECT int_id FROM ttrss_user_entries WHERE\n\t\t\t\t\t\t\t\tref_id = '{$ref_id}' AND owner_uid = '{$owner_uid}' AND\n\t\t\t\t\t\t\t\tfeed_id = '{$feed}' LIMIT 1"); if (db_num_rows($result) == 1) { $entry_int_id = db_fetch_result($result, 0, "int_id"); } } else { if ($debug_enabled) { _debug("update_rss_feed: user record FOUND"); } $entry_ref_id = db_fetch_result($result, 0, "ref_id"); $entry_int_id = db_fetch_result($result, 0, "int_id"); } if ($debug_enabled) { _debug("update_rss_feed: RID: {$entry_ref_id}, IID: {$entry_int_id}"); } $post_needs_update = false; $update_insignificant = false; if ($orig_num_comments != $num_comments) { $post_needs_update = true; $update_insignificant = true; } if ($content_hash != $orig_content_hash) { $post_needs_update = true; $update_insignificant = false; } if (db_escape_string($orig_title) != $entry_title) { $post_needs_update = true; $update_insignificant = false; } // if post needs update, update it and mark all user entries // linking to this post as updated if ($post_needs_update) { if (defined('DAEMON_EXTENDED_DEBUG')) { _debug("update_rss_feed: post {$entry_guid} needs update..."); } // print "<!-- post $orig_title needs update : $post_needs_update -->"; db_query($link, "UPDATE ttrss_entries\n\t\t\t\t\t\t\tSET title = '{$entry_title}', content = '{$entry_content}',\n\t\t\t\t\t\t\t\tcontent_hash = '{$content_hash}',\n\t\t\t\t\t\t\t\tupdated = '{$entry_timestamp_fmt}',\n\t\t\t\t\t\t\t\tnum_comments = '{$num_comments}'\n\t\t\t\t\t\t\tWHERE id = '{$ref_id}'"); if (!$update_insignificant) { if ($mark_unread_on_update) { db_query($link, "UPDATE ttrss_user_entries\n\t\t\t\t\t\t\t\t\tSET last_read = null, unread = true WHERE ref_id = '{$ref_id}'"); } else { if ($update_on_checksum_change) { db_query($link, "UPDATE ttrss_user_entries\n\t\t\t\t\t\t\t\t\tSET last_read = null WHERE ref_id = '{$ref_id}'\n\t\t\t\t\t\t\t\t\t\tAND unread = false"); } } } } } db_query($link, "COMMIT"); if ($debug_enabled) { _debug("update_rss_feed: assigning labels..."); } assign_article_to_labels($link, $entry_ref_id, $article_filters, $owner_uid); if ($debug_enabled) { _debug("update_rss_feed: looking for enclosures..."); } // enclosures $enclosures = array(); if ($use_simplepie) { $encs = $item->get_enclosures(); if (is_array($encs)) { foreach ($encs as $e) { $e_item = array($e->link, $e->type, $e->length); array_push($enclosures, $e_item); } } } else { // <enclosure> $e_ctr = $item['enclosure#']; if ($e_ctr > 0) { $e_item = array($item['enclosure@url'], $item['enclosure@type'], $item['enclosure@length']); array_push($enclosures, $e_item); for ($i = 0; $i <= $e_ctr; $i++) { if ($item["enclosure#{$i}@url"]) { $e_item = array($item["enclosure#{$i}@url"], $item["enclosure#{$i}@type"], $item["enclosure#{$i}@length"]); array_push($enclosures, $e_item); } } } // <media:content> // can there be many of those? yes -fox $m_ctr = $item['media']['content#']; if ($m_ctr > 0) { $e_item = array($item['media']['content@url'], $item['media']['content@medium'], $item['media']['content@length']); array_push($enclosures, $e_item); for ($i = 0; $i <= $m_ctr; $i++) { if ($item["media"]["content#{$i}@url"]) { $e_item = array($item["media"]["content#{$i}@url"], $item["media"]["content#{$i}@medium"], $item["media"]["content#{$i}@length"]); array_push($enclosures, $e_item); } } } } if ($debug_enabled) { _debug("update_rss_feed: article enclosures:"); print_r($enclosures); } db_query($link, "BEGIN"); foreach ($enclosures as $enc) { $enc_url = db_escape_string($enc[0]); $enc_type = db_escape_string($enc[1]); $enc_dur = db_escape_string($enc[2]); $result = db_query($link, "SELECT id FROM ttrss_enclosures\n\t\t\t\t\t\tWHERE content_url = '{$enc_url}' AND post_id = '{$entry_ref_id}'"); if (db_num_rows($result) == 0) { db_query($link, "INSERT INTO ttrss_enclosures\n\t\t\t\t\t\t\t(content_url, content_type, title, duration, post_id) VALUES\n\t\t\t\t\t\t\t('{$enc_url}', '{$enc_type}', '', '{$enc_dur}', '{$entry_ref_id}')"); } } db_query($link, "COMMIT"); // check for manual tags (we have to do it here since they're loaded from filters) foreach ($article_filters as $f) { if ($f[0] == "tag") { $manual_tags = trim_array(explode(",", $f[1])); foreach ($manual_tags as $tag) { if (tag_is_valid($tag)) { array_push($entry_tags, $tag); } } } } // Skip boring tags $boring_tags = trim_array(explode(",", mb_strtolower(get_pref($link, 'BLACKLISTED_TAGS', $owner_uid, ''), 'utf-8'))); $filtered_tags = array(); $tags_to_cache = array(); if ($entry_tags && is_array($entry_tags)) { foreach ($entry_tags as $tag) { if (array_search($tag, $boring_tags) === false) { array_push($filtered_tags, $tag); } } } $filtered_tags = array_unique($filtered_tags); if ($debug_enabled) { _debug("update_rss_feed: filtered article tags:"); print_r($filtered_tags); } // Save article tags in the database if (count($filtered_tags) > 0) { db_query($link, "BEGIN"); foreach ($filtered_tags as $tag) { $tag = sanitize_tag($tag); $tag = db_escape_string($tag); if (!tag_is_valid($tag)) { continue; } $result = db_query($link, "SELECT id FROM ttrss_tags\n\t\t\t\t\t\t\tWHERE tag_name = '{$tag}' AND post_int_id = '{$entry_int_id}' AND\n\t\t\t\t\t\t\towner_uid = '{$owner_uid}' LIMIT 1"); if ($result && db_num_rows($result) == 0) { db_query($link, "INSERT INTO ttrss_tags\n\t\t\t\t\t\t\t\t\t(owner_uid,tag_name,post_int_id)\n\t\t\t\t\t\t\t\t\tVALUES ('{$owner_uid}','{$tag}', '{$entry_int_id}')"); } array_push($tags_to_cache, $tag); } /* update the cache */ $tags_to_cache = array_unique($tags_to_cache); $tags_str = db_escape_string(join(",", $tags_to_cache)); db_query($link, "UPDATE ttrss_user_entries\n\t\t\t\t\t\tSET tag_cache = '{$tags_str}' WHERE ref_id = '{$entry_ref_id}'\n\t\t\t\t\t\tAND owner_uid = {$owner_uid}"); db_query($link, "COMMIT"); } if ($debug_enabled) { _debug("update_rss_feed: article processed"); } } if (!$last_updated) { if ($debug_enabled) { _debug("update_rss_feed: new feed, catching it up..."); } catchup_feed($link, $feed, false, $owner_uid); } if ($debug_enabled) { _debug("purging feed..."); } purge_feed($link, $feed, 0, $debug_enabled); db_query($link, "UPDATE ttrss_feeds\n\t\t\t\tSET last_updated = NOW(), last_error = '' WHERE id = '{$feed}'"); // db_query($link, "COMMIT"); } else { if ($use_simplepie) { $error_msg = mb_substr($rss->error(), 0, 250); } else { $error_msg = mb_substr(magpie_error(), 0, 250); } if ($debug_enabled) { _debug("update_rss_feed: error fetching feed: {$error_msg}"); } $error_msg = db_escape_string($error_msg); db_query($link, "UPDATE ttrss_feeds SET last_error = '{$error_msg}',\n\t\t\t\t\tlast_updated = NOW() WHERE id = '{$feed}'"); } if ($use_simplepie) { unset($rss); } if ($debug_enabled) { _debug("update_rss_feed: done"); } }
/** * Refresh the eye on design cache */ function ikit_social_remote_fetch_eyeondesign_rss() { $items = array(); global $g_options; $feed = new SimplePie(); $feed->set_cache_duration(0); $feed->set_timeout(5); $feed->set_feed_url(IKIT_SOCIAL_EYEONDESIGN_RSS_URL); $feed->set_cache_location(IKIT_DIR_CACHE); // XXX Fixes peculiar issue where feed was not pulling https://wcoaiga.atlassian.net/browse/AIGAIKIT-199 $feed->force_feed(true); $feed->force_fsockopen(true); $feed->init(); }
public static function getFeedData($params) { $rssurl = str_replace("\n", ",", JString::trim($params->get('rssurl', ''))); $feedTimeout = (int) $params->get('feedTimeout', 10); $rssdateorder = (int) $params->get('rssdateorder', 1); $dformat = $params->get('dformat', '%d %b %Y %H:%M %P'); $rssperfeed = (int) $params->get('rssperfeed', 3); $textfilter = JString::trim($params->get('textfilter', '')); $pagination = (int) $params->get('pagination', 0); $totalfeeds = (int) $params->get('rssitems', 5); $filtermode = (int) $params->get('filtermode', 0); $showfeedinfo = (int) $params->get('showfeedinfo', 1); $input_encoding = JString::trim($params->get('inputencoding', '')); $showimage = (int) $params->get('rssimage', 1); $cacheTime = (int) $params->get('feedcache', 15) * 60; //minutes $orderBy = $params->get('orderby', 'date'); $tmzone = (int) $params->get('tmzone', 0) ? true : false; $cachePath = JPATH_SITE . DS . 'cache' . DS . 'mod_we_ufeed_display'; $start = $end = 0; if ($pagination) { $pagination_items = (int) $params->get('paginationitems', 5); $current_limit = modFeedShowHelper::getPage($params->get('mid', 0)); $start = ($current_limit - 1) * $pagination_items; $end = $current_limit * $pagination_items; } #Get clean array $rss_urls = @array_filter(explode(",", $rssurl)); #If only 1 link, use totalfeeds for total limit if (count($rss_urls) == 1) { $rssperfeed = $totalfeeds; } # Intilize RSS Doc if (!class_exists('SimplePie')) { jimport('simplepie.simplepie'); } //Parser Code $simplepie = new SimplePie(); $simplepie->set_cache_location($cachePath); $simplepie->set_cache_duration($cacheTime); $simplepie->set_stupidly_fast(true); $simplepie->force_feed(true); //$simplepie->force_fsockopen(false); //gives priority to CURL if is installed $simplepie->set_timeout($feedTimeout); $simplepie->set_item_limit($rssperfeed); $simplepie->enable_order_by_date(false); if ($input_encoding) { $simplepie->set_input_encoding($input_encoding); $simplepie->set_output_encoding('UTF-8'); } $simplepie->set_feed_url($rss_urls); $simplepie->init(); $rssTotalItems = (int) $simplepie->get_item_quantity($totalfeeds); if ((int) $params->get('debug', 0)) { echo "<h3>Total RSS Items:" . $rssTotalItems . "</h3>"; echo print_r($simplepie, true); #debug } if (get_class($simplepie) != 'SimplePie' || !$rssTotalItems) { return array("rsstotalitems" => 0, 'items' => false); } $feedItems = array(); #store all feeds items $counter = 1; foreach ($simplepie->get_items($start, $end) as $key => $feed) { #Word Filter if (!empty($textfilter) && $filtermode != 0) { $filter = modFeedShowHelper::filterItems($feed, $textfilter); #Include #Exclude if ($filtermode == 1 && !$filter || $filtermode == 2 && $filter) { $rssTotalItems--; continue; #Include } } $FeedValues[$key] = new stdClass(); # channel header and link $channel = $feed->get_feed(); $FeedValues[$key]->FeedTitle = $channel->get_title(); $FeedValues[$key]->FeedLink = $channel->get_link(); if ($showfeedinfo) { $FeedValues[$key]->FeedFavicon = 'http://g.etfv.co/' . urlencode($FeedValues[$key]->FeedLink); } $FeedValues[$key]->FeedDescription = $channel->get_description(); $FeedValues[$key]->FeedLogo = $channel->get_image_url(); #Item $FeedValues[$key]->ItemTitle = $feed->get_title(); $feeDateUNIX = $feed->get_date('U'); if ($feeDateUNIX < 1) { $feeDateUNIX = strtotime(trim(str_replace(",", "", $feed->get_date('')))); } $FeedValues[$key]->ItemDate = WEJM16 ? JHTML::_('date', $feeDateUNIX, $dformat, $tmzone) : JHtml::date($feeDateUNIX, $dformat, $tmzone); $FeedValues[$key]->ItemLink = $feed->get_link(); //$feed->get_permalink(); $FeedValues[$key]->ItemText = $feed->get_description(); $FeedValues[$key]->ItemFulltext = $feed->get_content(); $FeedValues[$key]->ItemEnclosure = $feed->get_enclosure(); $FeedValues[$key]->ItemEnclosures = $feed->get_enclosures(); if ($showimage) { $FeedValues[$key]->ItemImage = ""; } //for next version if ($orderBy == 'title') { $idx = str_replace(array(':', '-', '(', ')', '+', '*', ' '), '', JString::strtolower(strip_tags($FeedValues[$key]->ItemTitle))); //ORDER BY TITLE } else { $idx = $feeDateUNIX; //Order By date } if (isset($feedItems[$idx])) { $idx .= $counter; } #unique idx $feedItems[$idx] = $FeedValues[$key]; $counter++; } if ($rssdateorder == 1) { krsort($feedItems); } elseif ($rssdateorder == 2) { ksort($feedItems); } elseif ($rssdateorder == 3) { shuffle($feedItems); } if ((int) $params->get('debug', 0)) { echo "<p>Total RSS in Array:" . count($feedItems) . "</p>"; } return array("rsstotalitems" => $rssTotalItems, 'items' => $feedItems); }
/** * Get an XML document * * @access public * @param string The type of xml parser needed 'DOM', 'RSS' or 'Simple' * @param array: * boolean ['lite'] When using 'DOM' if true or not defined then domit_lite is used * string ['rssUrl'] the rss url to parse when using "RSS" * string ['cache_time'] with 'RSS' - feed cache time. If not defined defaults to 3600 sec * @return object Parsed XML document object */ function &getXMLParser($type = 'DOM', $options = array()) { $doc = null; switch (strtolower($type)) { case 'rss': case 'atom': if (!is_null($options['rssUrl'])) { jimport('simplepie.simplepie'); if (!is_writable(JPATH_BASE . DS . 'cache')) { $options['cache_time'] = 0; } $simplepie = new SimplePie($options['rssUrl'], JPATH_BASE . DS . 'cache', isset($options['cache_time']) ? $options['cache_time'] : 0); $simplepie->force_feed(true); $simplepie->handle_content_type(); if ($simplepie->init()) { $doc = $simplepie; } else { JError::raiseWarning('SOME_ERROR_CODE', JText::_('ERROR LOADING FEED DATA')); } } break; case 'simple': jimport('joomla.utilities.simplexml'); $doc = new JSimpleXML(); break; case 'dom': default: if (!isset($options['lite']) || $options['lite']) { jimport('domit.xml_domit_lite_include'); $doc = new DOMIT_Lite_Document(); } else { jimport('domit.xml_domit_include'); $doc = new DOMIT_Document(); } } return $doc; }
public static function getFeedParser($url, $cache_time = 0) { mimport('simplepie.simplepie'); $cache = self::getCache('feed_parser', 'callback'); if ($cache_time > 0) { $cache->setLifeTime($cache_time); } $simplepie = new SimplePie(null, null, 0); $simplepie->enable_cache(false); $simplepie->set_feed_url($url); $simplepie->force_feed(true); $contents = $cache->get(array($simplepie, 'init'), null, false, false); if ($contents) { return $simplepie; } else { MError::raiseWarning('SOME_ERROR_CODE', MText::_('MLIB_UTIL_ERROR_LOADING_FEED_DATA')); } return false; }
SiteConfig::use_apc($options->apc); $extractor->fingerprints = $options->fingerprints; $extractor->allowedParsers = $options->allowed_parsers; //////////////////////////////// // Get RSS/Atom feed //////////////////////////////// if (!$html_only) { debug('--------'); debug("Attempting to process URL as feed"); // Send user agent header showing PHP (prevents a HTML response from feedburner) $http->userAgentDefault = HumbleHttpAgent::UA_PHP; // configure SimplePie HTTP extension class to use our HumbleHttpAgent instance SimplePie_HumbleHttpAgent::set_agent($http); $feed = new SimplePie(); // some feeds use the text/html content type - force_feed tells SimplePie to process anyway $feed->force_feed(true); $feed->set_file_class('SimplePie_HumbleHttpAgent'); //$feed->set_feed_url($url); // colons appearing in the URL's path get encoded $feed->feed_url = $url; $feed->set_autodiscovery_level(SIMPLEPIE_LOCATOR_NONE); $feed->set_timeout(20); $feed->enable_cache(false); $feed->set_stupidly_fast(true); $feed->enable_order_by_date(false); // we don't want to do anything to the feed $feed->set_url_replacements(array()); // initialise the feed // the @ suppresses notices which on some servers causes a 500 internal server error $result = @$feed->init(); //$feed->handle_content_type(); //$feed->get_title();
/** * Get a parsed XML Feed Source * * @since: 1.6 * @static * @param string url for feed source * @param int time to cache feed for (using internal cache mechanism) * @return mixed Parsed SimplePie object on success, false on failure */ public static function getFeedParser($url, $cache_time = 0) { jimport('simplepie.simplepie'); if (!is_writable(JPATH_CACHE)) { $cache_time = 0; } $simplepie = new SimplePie($url, JPATH_CACHE, $cache_time); $simplepie->force_feed(true); if ($simplepie->init()) { return $simplepie; } else { JError::raiseWarning('SOME_ERROR_CODE', JText::_('ERROR_LOADING_FEED_DATA')); } return false; }