/**
* Process RSS feed and return results.
*
* @param $feed_url
* @param null $cache_name
* @param int $cache_expires
* @return array|mixed
*/
public static function getNewsFeed($feed_url, $cache_name = NULL, $cache_expires = 900)
{
if (!is_null($cache_name)) {
$feed_cache = Cache::get('feed_' . $cache_name);
} else {
$feed_cache = null;
}
if ($feed_cache) {
return $feed_cache;
}
// Catch the occasional error when the RSS feed is malformed or the HTTP request times out.
try {
$news_feed = Reader::import($feed_url);
} catch (\Exception $e) {
$news_feed = NULL;
}
if (!is_null($news_feed)) {
$latest_news = array();
$article_num = 0;
foreach ($news_feed as $item) {
$article_num++;
$news_item = array('num' => $article_num, 'title' => $item->getTitle(), 'timestamp' => $item->getDateModified()->getTimestamp(), 'description' => trim($item->getDescription()), 'link' => $item->getLink(), 'categories' => $item->getCategories()->getValues());
$latest_news[] = $news_item;
}
$latest_news = array_slice($latest_news, 0, 10);
if (!is_null($cache_name)) {
Cache::set($latest_news, 'feed_' . $cache_name, array('feeds', $cache_name), $cache_expires);
}
return $latest_news;
}
}
/**
* Loads a newsfeed object.
*
* @param string $feedurl
* @param bool $cache
* @return Reader
*/
public function load($url, $cache = true)
{
if ($cache) {
Reader::setCache(new ZendCacheDriver('cache/expensive'));
}
$feed = Reader::import($url);
return $feed;
}
public function read() : array
{
$url = sprintf(self::ATOM_FORMAT, $this->user);
$feed = FeedReader::import($url);
$entries = Collection::create($feed)->filterChain($this->filters)->slice($this->limit)->map(function ($entry) {
return ['title' => $entry->getTitle(), 'link' => $entry->getLink()];
});
return ['last_modified' => $feed->getDateModified(), 'link' => $feed->getLink(), 'links' => $entries->toArray()];
}
/**
* Loads feed from an url or file path
*
* @param string $file
*
* @return Reader
*/
public function load($file)
{
if (file_exists($file)) {
$this->feed = ZendReader::importFile($file);
} else {
$this->feed = ZendReader::import($file);
}
return $this;
}
/**
* Loads a newsfeed object.
*
* @param string $feedurl
* @param int $cache - number of seconds to cache the RSS feed data for
* @return Reader
*/
public function load($url, $cache = 3600)
{
if ($cache !== false) {
Reader::setCache(new ZendCacheDriver('cache/expensive', $cache));
}
// Load the RSS feed, either from remote URL or from cache
// (if specified above and still fresh)
$feed = Reader::import($url);
return $feed;
}
/**
* Simple utility function which imports any feed URL and
* determines the existence of Hub Server endpoints. This works
* best if directly given an instance of Zend_Feed_Reader_Atom|Rss
* to leverage off.
*
* @param \Zend\Feed\Reader\Feed\AbstractFeed|string $source
* @return array
* @throws Exception\InvalidArgumentException
*/
public static function detectHubs($source)
{
if (is_string($source)) {
$feed = Reader\Reader::import($source);
} elseif ($source instanceof Reader\Feed\AbstractFeed) {
$feed = $source;
} else {
throw new Exception\InvalidArgumentException('The source parameter was' . ' invalid, i.e. not a URL string or an instance of type' . ' Zend\\Feed\\Reader\\Feed\\AbstractFeed');
}
return $feed->getHubs();
}
public function run()
{
$feed = Reader::import($this->feedUrl);
$data = array('title' => $feed->getTitle(), 'link' => $feed->getLink(), 'dateModified' => $feed->getDateModified(), 'description' => $feed->getDescription(), 'language' => $feed->getLanguage(), 'entries' => array());
foreach ($feed as $entry) {
$edata = array('title' => $entry->getTitle(), 'description' => $entry->getDescription(), 'dateModified' => $entry->getDateModified(), 'authors' => $entry->getAuthors(), 'link' => $entry->getLink(), 'content' => $entry->getContent());
$data['entries'][] = $edata;
}
echo $this->render('default', ['data' => $data]);
// $this->registerClientScript();
}
public function rssProxyAction()
{
$overrideFeedUrl = $this->getEvent()->getRequest()->getQuery()->get('urlOverride');
$limit = $this->getEvent()->getRequest()->getQuery()->get('limit');
$instanceId = $this->getEvent()->getRequest()->getQuery()->get('instanceId');
/** @var \Rcm\Service\PluginManager $pluginManager */
$pluginManager = $this->serviceLocator->get('\\Rcm\\Service\\PluginManager');
if ($instanceId > 0) {
$instanceConfig = $pluginManager->getInstanceConfig($instanceId);
} else {
$instanceConfig = $pluginManager->getDefaultInstanceConfig('RcmRssFeed');
}
$feedUrl = $instanceConfig['rssFeedUrl'];
$cacheKey = 'rcmrssfeed-' . md5($feedUrl);
if ($this->cacheMgr->hasItem($cacheKey)) {
$viewRssData = json_decode($this->cacheMgr->getItem($cacheKey));
$this->sendJson($viewRssData);
}
if (!empty($overrideFeedUrl) && $overrideFeedUrl != 'null') {
//$permissions = $this->userMgr->getLoggedInAdminPermissions();
$permissions = null;
/**
* Only admins can override the url. This prevents people from using
* our proxy to DDOS other sites.
*/
$allowed = $this->rcmIsAllowed('sites.' . $this->siteId, 'admin');
if ($allowed) {
$feedUrl = $overrideFeedUrl;
}
}
if (empty($limit)) {
$limit = $instanceConfig['rssFeedLimit'];
}
$rssReader = new Reader();
//Tried to add a timeout like this but it didnt work
$httpClient = new Client($feedUrl, ['timeout' => 5]);
$rssReader->setHttpClient($httpClient);
try {
$feedData = $rssReader->import($feedUrl);
} catch (\Exception $e) {
$feedData = [];
}
$feedCount = 0;
$viewRssData = [];
foreach ($feedData as $entry) {
if ($feedCount == $limit) {
break;
}
$viewRssData[] = ['feedtitle' => $entry->getTitle(), 'description' => $entry->getDescription(), 'dateModified' => $entry->getDateModified(), 'authors' => $entry->getAuthors(), 'feedlink' => $entry->getLink()];
$feedCount++;
}
$this->cacheMgr->addItem($cacheKey, json_encode($viewRssData));
$this->sendJson($viewRssData);
}
/**
* @return array
*/
public function getEntries()
{
if (empty($this->entries)) {
$feeds = $this->getOutlet()->getFeeds();
foreach ($feeds as $feed) {
$reader = Reader::import($feed->getUrl());
foreach ($reader as $entry) {
$this->entries[] = $entry;
}
}
}
return $this->entries;
}
public function import()
{
$this->logger('Import ' . $this->uri, 'info', $this->logger_level);
try {
$this->feed = Reader::import($this->uri);
if (!isset($this->feed)) {
throw new \Exception('Unreadble');
}
} catch (\Exception $e) {
$this->logger('Feed empty ', 'err', $this->logger_level);
return false;
}
}
/**
* Simple utility function which imports any feed URL and
* determines the existence of Hub Server endpoints. This works
* best if directly given an instance of Zend_Feed_Reader_Atom|Rss
* to leverage off.
*
* @param Zend_Feed_Reader_FeedAbstract|\Zend\Feed\AbstractFeed|string $source
* @return array
*/
public static function detectHubs($source)
{
if (is_string($source)) {
$feed = Reader\Reader::import($source);
} elseif (is_object($source) && $source instanceof Reader\FeedAbstract) {
$feed = $source;
} elseif (is_object($source) && $source instanceof \Zend\Feed\AbstractFeed) {
$feed = Reader\Reader::importFeed($source);
} else {
require_once 'Zend/Feed/Pubsubhubbub/Exception.php';
throw new Exception('The source parameter was' . ' invalid, i.e. not a URL string or an instance of type' . ' Zend_Feed_Reader_FeedAbstract or Zend_Feed_Abstract');
}
return $feed->getHubs();
}
/**
* @inheritDoc
*
*/
public function import()
{
if (empty($this->feedUri) || !is_string($this->feedUri)) {
throw new Exception\UnexpectedValueException(sprintf('Feed uri not valid.'));
}
$feed = Reader::import($this->feedUri);
foreach ($feed as $entry) {
if ($this->entryIsNew($entry)) {
$postId = $this->createPostFromEntry($entry);
if (!empty($this->entryParams['taxonomy'])) {
$this->createTermsFromEntry($postId, $entry);
}
}
}
}
public function getData()
{
// Fetch the latest Slashdot headlines
try {
$slashdotRss = \Zend\Feed\Reader\Reader::import('http://rss.slashdot.org/Slashdot/slashdot');
} catch (\Zend\Feed\Exception\Reader\RuntimeException $e) {
// feed import failed
echo "Exception caught importing feed: {$e->getMessage()}\n";
exit;
}
// Initialize the channel/feed data array
$channel = array('title' => $slashdotRss->getTitle(), 'link' => $slashdotRss->getLink(), 'description' => $slashdotRss->getDescription(), 'items' => array());
// Loop over each channel item/entry and store relevant data for each
foreach ($slashdotRss as $item) {
$channel['items'][] = array('title' => $item->getTitle(), 'link' => $item->getLink(), 'description' => $item->getDescription());
}
return $channel;
}
public static function fetch($feed_url, $params = array())
{
try {
$news_feed = Reader::import($feed_url);
} catch (\Exception $e) {
return array();
}
if (is_null($news_feed)) {
return array();
}
$latest_news = array();
$article_num = 0;
foreach ($news_feed as $item) {
$article_num++;
$guid = $item->getId();
$title = $item->getTitle();
// Process categories.
$categories_raw = $item->getCategories()->getValues();
// Process main description.
$description = trim($item->getDescription());
// Remove extraneous tags.
$description = str_replace(array("\r", "\n"), array('', ' '), $description);
// Strip new lines.
$description = preg_replace('/<a[^(>)]+>read more<\\/a>/iu', '', $description);
// Remove "read more" link.
$web_url = $item->getLink();
if (is_array($web_url)) {
$web_url = $web_url[0];
}
if (!$web_url && substr($guid, 0, 4) == 'http') {
$web_url = $guid;
}
$author = $item->getAuthor();
if (is_array($author)) {
$author = $author[0]->nodeValue;
}
$news_item = array('guid' => 'rss_' . md5($guid), 'timestamp' => $item->getDateModified()->getTimestamp(), 'media_format' => 'mixed', 'title' => $title, 'body' => $description, 'web_url' => $web_url, 'author' => $author);
$latest_news[] = $news_item;
}
return $latest_news;
}
public function read()
{
$url = sprintf(self::ATOM_FORMAT, $this->user);
$feed = FeedReader::import($url);
$lastModified = $feed->getDateModified();
$altLink = $feed->getLink();
$entries = array();
$i = 0;
foreach ($feed as $entry) {
if (!$this->filter($entry)) {
continue;
}
$data = array('title' => $entry->getTitle(), 'link' => $entry->getLink());
$entries[] = $data;
$i++;
if ($i > $this->limit) {
break;
}
}
return array('last_modified' => $lastModified, 'link' => $altLink, 'links' => $entries);
}
public function refreshFeed() : array
{
$cacheId = $this->blogOptions->getCacheKey();
$feed = Reader::import($this->blogOptions->getFeed());
$feed = $this->processFeed($feed);
// If no feed has been cached yet, cache current one and return
if (!$this->feedCache->contains($cacheId)) {
$this->templatesCache->deleteAll();
$this->feedCache->save($cacheId, $feed);
return $feed;
}
// Check if the last feed has changed, otherwise, return
$cachedFeed = $this->feedCache->fetch($cacheId);
if ($cachedFeed[0]['link'] === $feed[0]['link']) {
return $feed;
}
// If the feed has changed, clear all cached elements so that views are refreshed, and cache feed too
$this->templatesCache->deleteAll();
$this->feedCache->save($cacheId, $feed);
return $feed;
}
/**
* Aucune annotation Route n'a été définie pour cette méthode de controller,
* la page ne sera pas directement accessible.
*
* La méthode peut être appelée depuis un fichier de template
* {{ render(controller('MmiBlogBundle:Feed:last', {limit: 5})) }}
*/
public function lastAction($limit)
{
//import d'un flux RSS
try {
$reader = Reader::import('http://etin.yourphototravel.com/fr/etins.rss');
} catch (\Exception $e) {
$reader = array();
}
//lecture du flux RSS et préparation des paramètres à envoyer à la vue
$items = array();
$count = 0;
foreach ($reader as $item) {
$crawler = new Crawler($item->getDescription());
$items[] = array('title' => $item->getTitle(), 'link' => $item->getLink(), 'image' => $crawler->filter('img')->first()->attr('src'), 'description' => utf8_decode($crawler->text()));
$count++;
if ($count == $limit) {
break;
}
}
//création de la réponse
return $this->get('templating')->renderResponse('MmiBlogBundle:Feed:list.html.twig', array('items' => $items));
}
/**
* Execute the console command.
*
* @return void
* @throws \RuntimeException
* @throws \Exception
*/
public function fire()
{
$sourceId = $this->argument('source');
$sources = new Collection();
if ($sourceId !== null && is_numeric($sourceId)) {
$source = Source::find($sourceId);
if ($source && $source->getAttribute('is_enabled')) {
$sources = new Collection([$source]);
}
} else {
$sources = Source::query()->where('is_enabled', '=', 1)->get();
}
$sources->each(function (Source $source) {
try {
$channel = Reader::import($source->getAttribute('source_url'));
$maxItems = $source->getAttribute('max_items');
$this->getOutput()->writeln($channel->getTitle());
$itemCount = 0;
/** @type Rss $item */
foreach ($channel as $item) {
++$itemCount;
$this->getOutput()->writeln($itemCount . '. ' . $item->getTitle());
$attributes = ['item_id' => $item->getId(), 'source_id' => $source->getAttribute('id'), 'title' => $item->getTitle(), 'link' => $item->getLink(), 'description' => strip_tags($item->getDescription()), 'category' => implode(', ', $item->getCategories()->getValues()), 'comments' => $item->getCommentLink(), 'pub_date' => $item->getDateCreated(), 'is_published' => $source->getAttribute('publish_new_items')];
if ($item->getAuthors() !== null && is_array($item->getAuthors())) {
$attributes['author'] = implode(', ', $item->getAuthors());
}
Item::firstOrCreate($attributes);
if ($maxItems > 0 && $itemCount >= $maxItems) {
break;
}
}
$source->setAttribute('fetched_at', new Carbon());
$source->save();
} catch (\Exception $e) {
$this->getOutput()->writeln('<error>' . $e->getMessage() . '</error>');
}
});
}
/**
* @param string $url
* @param int $limit
* @return array
*/
public function read($url, $limit = 5)
{
$data = [];
$this->client->setUri($url)->setOptions(['adapter' => 'Zend\\Http\\Client\\Adapter\\Curl']);
try {
\Zend\Feed\Reader\Reader::setHttpClient($this->client);
$feed = \Zend\Feed\Reader\Reader::import($url);
if ($feed) {
$i = 0;
foreach ($feed as $entry) {
$edata = ['title' => $entry->getTitle(), 'description' => preg_replace("/<img(.+?)>/", "", $entry->getDescription()), 'dateModified' => $entry->getDateModified(), 'authors' => $entry->getAuthors(), 'link' => $entry->getLink(), 'content' => $entry->getContent()];
$data[] = (object) $edata;
$i++;
if ($i == $limit) {
break;
}
}
}
} catch (RuntimeException $exception) {
$data[] = [];
}
return $data;
}
public function processFeedsAction()
{
$request = $this->getRequest();
$verbose = $request->getParam('verbose') || $request->getParam('v');
$userFeedsTable = $this->getTable('UserFeedsTable');
$userFeedArticlesTable = $this->getTable('UserFeedArticlesTable');
$feeds = $userFeedsTable->select();
foreach ($feeds as $feed) {
if ($verbose) {
printf("Processing feed: %s\n", $feed['url']);
}
$lastUpdate = strtotime($feed['updated_at']);
$rss = Reader::import($feed['url']);
// Loop over each channel item/entry and store relevant data for each
foreach ($rss as $item) {
$timestamp = $item->getDateCreated()->getTimestamp();
if ($timestamp > $lastUpdate) {
if ($verbose) {
printf("Processing item: %s\n", $item->getTitle());
}
$author = $item->getAuthor();
if (is_array($author)) {
$author = $author['name'];
}
$userFeedArticlesTable->create($feed['id'], $item->getTitle(), $item->getContent(), $item->getLink(), $author);
}
}
if ($verbose) {
echo "Updating timestamp\n";
}
$userFeedsTable->updateTimestamp($feed['id']);
if ($verbose) {
echo "Finished feed processing\n\n";
}
}
}
/**
* Supports lazy loading of feeds using Zend_Feed_Reader::import() but
* delegates any other operations to the parent class.
*
* @param string $offset
* @return mixed
*/
public function offsetGet($offset)
{
if ($offset == 'feed' && !$this->offsetExists('feed')) {
if (!$this->offsetExists('href')) {
return null;
}
$feed = Reader::import($this->offsetGet('href'));
$this->offsetSet('feed', $feed);
return $feed;
}
return parent::offsetGet($offset);
}
/**
* process
*
* Called after the Search Results object has performed its main search. This
* may be used to extract necessary information from the Search Results object
* or to perform completely unrelated processing.
*
* @param \VuFind\Search\Base\Results $results Search results object
*
* @return void
*/
public function process($results)
{
FeedReader::setHttpClient(new \VuFind\Http\Client());
$parsedFeed = FeedReader::import($this->targetUrl);
$resultsProcessed = array();
foreach ($parsedFeed as $key => $value) {
$link = (string) $value->link;
if (!empty($link)) {
$resultsProcessed[] = array('title' => (string) $value->title, 'link' => substr($link, 0, strpos($link, '.srw')) . '.html', 'enclosure' => (string) $value->enclosure['url']);
}
if (count($resultsProcessed) == $this->limit) {
break;
}
}
if (!empty($resultsProcessed)) {
$this->results = array('worksArray' => $resultsProcessed, 'feedTitle' => $this->searchSite, 'sourceLink' => $this->sitePath);
} else {
$this->results = false;
}
}
/**
* Utility function for processing a feed (see readFeed, readFeedFromUrl).
*
* @param array $feedConfig Configuration
* @param Zend\Mvc\Controller\Plugin\Url $urlHelper Url helper
* @param string $viewUrl View URL
* @param string $id Feed id (needed when the
* feed content is shown on a content page or in a modal)
*
* @return mixed null|array
*/
protected function processReadFeed($feedConfig, $urlHelper, $viewUrl, $id = null)
{
$config = $feedConfig['result'];
$url = $feedConfig['url'];
$type = $config->type;
$cacheKey = $config->toArray();
$cacheKey['language'] = $this->translator->getLocale();
$modal = false;
$showFullContentOnSite = isset($config->linkTo) && in_array($config->linkTo, ['modal', 'content-page']);
$modal = $config->linkTo == 'modal';
$contentPage = $config->linkTo == 'content-page';
$dateFormat = isset($config->dateFormat) ? $config->dateFormat : 'j.n.';
$contentDateFormat = isset($config->contentDateFormat) ? $config->contentDateFormat : 'j.n.Y';
$itemsCnt = isset($config->items) ? $config->items : null;
$elements = isset($config->content) ? $config->content : [];
$channel = null;
// Check for cached version
$cacheDir = $this->cacheManager->getCache('feed')->getOptions()->getCacheDir();
$localFile = "{$cacheDir}/" . md5(var_export($cacheKey, true)) . '.xml';
$maxAge = isset($this->mainConfig->Content->feedcachetime) ? $this->mainConfig->Content->feedcachetime : 10;
Reader::setHttpClient($this->http->createClient());
if ($maxAge) {
if (is_readable($localFile) && time() - filemtime($localFile) < $maxAge * 60) {
$channel = Reader::importFile($localFile);
}
}
if (!$channel) {
// No cache available, read from source.
if (preg_match('/^http(s)?:\\/\\//', $url)) {
// Absolute URL
$channel = Reader::import($url);
} else {
if (substr($url, 0, 1) === '/') {
// Relative URL
$url = substr($viewUrl, 0, -1) . $url;
try {
$channel = Reader::import($url);
} catch (\Exception $e) {
$this->logError("Error importing feed from url {$url}");
$this->logError(" " . $e->getMessage());
}
} else {
// Local file
if (!is_file($url)) {
$this->logError("File {$url} could not be found");
throw new \Exception('Error reading feed');
}
$channel = Reader::importFile($url);
}
}
if ($channel) {
file_put_contents($localFile, $channel->saveXml());
}
}
if (!$channel) {
return false;
}
$content = ['title' => 'getTitle', 'text' => 'getContent', 'image' => 'getEnclosure', 'link' => 'getLink', 'date' => 'getDateCreated', 'contentDate' => 'getDateCreated'];
$xpathContent = ['html' => '//item/content:encoded'];
$items = [];
$cnt = 0;
$xpath = null;
$cnt = 0;
foreach ($channel as $item) {
if (!$xpath) {
$xpath = $item->getXpath();
}
$data = [];
$data['modal'] = $modal;
foreach ($content as $setting => $method) {
if (!isset($elements[$setting]) || $elements[$setting] != 0) {
$value = $item->{$method}();
if (is_object($value)) {
$value = get_object_vars($value);
}
if ($setting == 'image') {
if (!$value || stripos($value['type'], 'image') === false) {
// Attempt to parse image URL from content
if ($value = $this->extractImage($item->getContent())) {
$value = ['url' => $value];
}
}
} else {
if ($setting == 'date') {
if (isset($value['date'])) {
$value = new \DateTime($value['date']);
if ($dateFormat) {
$value = $value->format($dateFormat);
}
}
} else {
if ($setting == 'contentDate') {
if (isset($value['date'])) {
$value = new \DateTime($value['date']);
if ($contentDateFormat) {
$value = $value->format($contentDateFormat);
}
}
} else {
if ($setting == 'link' && $showFullContentOnSite) {
$link = $urlHelper->fromRoute('feed-content-page', ['page' => $id, 'element' => $cnt]);
$value = $link;
} else {
if (is_string($value)) {
$value = strip_tags($value);
}
}
}
}
}
if ($value) {
$data[$setting] = $value;
}
}
}
// Make sure that we have something to display
$accept = $data['title'] && trim($data['title']) != '' || $data['text'] && trim($data['text']) != '' || $data['image'];
if (!$accept) {
continue;
}
$items[] = $data;
$cnt++;
if ($itemsCnt !== null && $cnt == $itemsCnt) {
break;
}
}
if ($xpath && !empty($xpathContent)) {
if ($xpathItem = $xpath->query('//item/content:encoded')->item(0)) {
$contentSearch = isset($config->htmlContentSearch) ? $config->htmlContentSearch->toArray() : [];
$contentReplace = isset($config->htmlContentReplace) ? $config->htmlContentReplace->toArray() : [];
$searchReplace = array_combine($contentSearch, $contentReplace);
$cnt = 0;
foreach ($items as &$item) {
foreach ($xpathContent as $setting => $xpathElement) {
$content = $xpath->query($xpathElement, $xpathItem)->item($cnt++)->nodeValue;
// Remove width & height declarations from style
// attributes in div & p elements
$dom = new \DOMDocument();
libxml_use_internal_errors(true);
$dom->loadHTML(mb_convert_encoding($content, 'HTML-ENTITIES', 'UTF-8'));
$domx = new \DOMXPath($dom);
$elements = $domx->query('//div[@style]|//p[@style]');
foreach ($elements as $el) {
$styleProperties = [];
$styleAttr = $el->getAttribute('style');
$properties = explode(';', $styleAttr);
foreach ($properties as $prop) {
list($field, $val) = explode(':', $prop);
if (stristr($field, 'width') === false && stristr($field, 'height') === false && stristr($field, 'margin') === false) {
$styleProperties[] = $prop;
}
}
$el->removeAttribute("style");
$el->setAttribute('style', implode(';', $styleProperties));
}
$content = $dom->saveHTML();
// Process feed specific search-replace regexes
foreach ($searchReplace as $search => $replace) {
$pattern = "/{$search}/";
$replaced = preg_replace($pattern, $replace, $content);
if ($replaced !== null) {
$content = $replaced;
}
}
$item[$setting] = $content;
}
}
}
}
return compact('channel', 'items', 'config', 'modal', 'contentPage');
}
/**
* Add a new subscription
*
* @return JsonModel
*/
public function create($data)
{
$username = $this->params()->fromRoute('username');
$usersTable = $this->getTable('UsersTable');
$user = $usersTable->getByUsername($username);
$userFeedsTable = $this->getTable('UserFeedsTable');
$rssLinkXpath = '//link[@type="application/rss+xml"]';
$faviconXpath = '//link[@rel="shortcut icon"]';
$client = new Client($data['url']);
$client->setEncType(Client::ENC_URLENCODED);
$client->setMethod(\Zend\Http\Request::METHOD_GET);
$response = $client->send();
if ($response->isSuccess()) {
$html = $response->getBody();
$html = mb_convert_encoding($html, 'HTML-ENTITIES', "UTF-8");
$dom = new Query($html);
$rssUrl = $dom->execute($rssLinkXpath);
if (!count($rssUrl)) {
throw new Exception('Rss url not found in the url provided', 404);
}
$rssUrl = $rssUrl->current()->getAttribute('href');
$faviconUrl = $dom->execute($faviconXpath);
if (count($faviconUrl)) {
$faviconUrl = $faviconUrl->current()->getAttribute('href');
} else {
$faviconUrl = null;
}
} else {
throw new Exception("Website not found", 404);
}
$rss = Reader::import($rssUrl);
return new JsonModel(array('result' => $userFeedsTable->create($user->id, $rssUrl, $rss->getTitle(), $faviconUrl)));
}
/**
* @group ZF-8328
*/
public function testImportsUriAndThrowsExceptionIfNotAFeed()
{
$this->setExpectedException('Zend\\Feed\\Exception');
if (!defined('TESTS_ZEND_FEED_READER_ONLINE_ENABLED') || !constant('TESTS_ZEND_FEED_READER_ONLINE_ENABLED')) {
$this->markTestSkipped('testImportsUri() requires a network connection');
return;
}
$feed = Reader\Reader::import('http://twitter.com/alganet');
}
/**
* Called after the Search Results object has performed its main search. This
* may be used to extract necessary information from the Search Results object
* or to perform completely unrelated processing.
*
* @param \VuFind\Search\Base\Results $results Search results object
*
* @return void
*/
public function process($results)
{
$this->debug('Pulling feed from ' . $this->targetUrl);
if (null !== $this->httpService) {
FeedReader::setHttpClient($this->httpService->createClient($this->targetUrl));
}
$parsedFeed = FeedReader::import($this->targetUrl);
$resultsProcessed = [];
foreach ($parsedFeed as $value) {
$link = $value->getLink();
if (!empty($link)) {
$resultsProcessed[] = ['title' => $value->getTitle(), 'link' => substr($link, 0, strpos($link, '.srw')) . '.html', 'enclosure' => $value->getEnclosure()['url']];
}
if (count($resultsProcessed) == $this->limit) {
break;
}
}
if (!empty($resultsProcessed)) {
$this->results = ['worksArray' => $resultsProcessed, 'feedTitle' => $this->searchSite, 'sourceLink' => $this->sitePath];
} else {
$this->results = false;
}
}
/**
* @group ZF-8328
* @expectedException Zend\Feed\Reader\Exception\RuntimeException
*/
public function testImportsUriAndThrowsExceptionIfNotAFeed()
{
if (!constant('TESTS_ZEND_FEED_READER_ONLINE_ENABLED')) {
$this->markTestSkipped('testImportsUri() requires a network connection');
}
Reader\Reader::import('http://twitter.com/alganet');
}
* obtain it through the world-wide-web, please send an email
* to license@zend.com so we can send you a copy immediately.
*
* @category Zend
* @package Zend_Feed
* @subpackage Demos
* @copyright Copyright (c) 2005-2012 Zend Technologies USA Inc. (http://www.zend.com)
* @license http://framework.zend.com/license/new-bsd New BSD License
*/
use Zend\Loader\StandardAutoloader;
use Zend\Feed\Reader\Reader;
/**
* Consume an RSS feed and display all of the titles and
* associated links within.
*/
require_once dirname(dirname(dirname(__DIR__))).'/library/Zend/Loader/StandardAutoloader.php';
$loader = new StandardAutoloader;
$loader->register();
$rss = Reader::import('http://news.google.com/?output=rss');
foreach ($rss as $item) {
echo "<p>" . $item->getTitle() . "<br />", "\n";
echo $item->getLink() . "</p>";
}
/**
* Return feed content and settings in JSON format.
*
* @return mixed
*/
public function getFeedAjax()
{
if (!($id = $this->params()->fromQuery('id'))) {
return $this->output('Missing feed id', self::STATUS_ERROR);
}
$touchDevice = $this->params()->fromQuery('touch-device') !== null ? $this->params()->fromQuery('touch-device') === '1' : false;
$config = $this->getServiceLocator()->get('VuFind\\Config')->get('rss');
if (!isset($config[$id])) {
return $this->output('Missing feed configuration', self::STATUS_ERROR);
}
$config = $config[$id];
if (!$config->active) {
return $this->output('Feed inactive', self::STATUS_ERROR);
}
if (!($url = $config->url)) {
return $this->output('Missing feed URL', self::STATUS_ERROR);
}
$translator = $this->getServiceLocator()->get('VuFind\\Translator');
$language = $translator->getLocale();
if (isset($url[$language])) {
$url = trim($url[$language]);
} else {
if (isset($url['*'])) {
$url = trim($url['*']);
} else {
return $this->output('Missing feed URL', self::STATUS_ERROR);
}
}
$type = $config->type;
$channel = null;
// Check for cached version
$cacheEnabled = false;
$cacheDir = $this->getServiceLocator()->get('VuFind\\CacheManager')->getCache('feed')->getOptions()->getCacheDir();
$cacheKey = $config->toArray();
$cacheKey['language'] = $language;
$localFile = "{$cacheDir}/" . md5(var_export($cacheKey, true)) . '.xml';
$cacheConfig = $this->getServiceLocator()->get('VuFind\\Config')->get('config');
$maxAge = isset($cacheConfig->Content->feedcachetime) ? $cacheConfig->Content->feedcachetime : false;
$httpService = $this->getServiceLocator()->get('VuFind\\Http');
Reader::setHttpClient($httpService->createClient());
if ($maxAge) {
$cacheEnabled = true;
if (is_readable($localFile) && time() - filemtime($localFile) < $maxAge * 60) {
$channel = Reader::importFile($localFile);
}
}
if (!$channel) {
// No cache available, read from source.
if (preg_match('/^http(s)?:\\/\\//', $url)) {
// Absolute URL
$channel = Reader::import($url);
} else {
if (substr($url, 0, 1) === '/') {
// Relative URL
$url = substr($this->getServerUrl('home'), 0, -1) . $url;
$channel = Reader::import($url);
} else {
// Local file
if (!is_file($url)) {
return $this->output("File {$url} could not be found", self::STATUS_ERROR);
}
$channel = Reader::importFile($url);
}
}
}
if (!$channel) {
return $this->output('Parsing failed', self::STATUS_ERROR);
}
if ($cacheEnabled) {
file_put_contents($localFile, $channel->saveXml());
}
$content = ['title' => 'getTitle', 'text' => 'getContent', 'image' => 'getEnclosure', 'link' => 'getLink', 'date' => 'getDateCreated'];
$dateFormat = isset($config->dateFormat) ? $config->dateFormat : 'j.n.';
$itemsCnt = isset($config->items) ? $config->items : null;
$items = [];
foreach ($channel as $item) {
$data = [];
foreach ($content as $setting => $method) {
if (!isset($config->content[$setting]) || $config->content[$setting] != 0) {
$tmp = $item->{$method}();
if (is_object($tmp)) {
$tmp = get_object_vars($tmp);
}
if ($setting == 'image') {
if (!$tmp || stripos($tmp['type'], 'image') === false) {
// Attempt to parse image URL from content
if ($tmp = $this->extractImage($item->getContent())) {
$tmp = ['url' => $tmp];
}
}
} else {
if ($setting == 'date') {
if (isset($tmp['date'])) {
$tmp = new \DateTime($tmp['date']);
$tmp = $tmp->format($dateFormat);
}
} else {
if (is_string($tmp)) {
$tmp = strip_tags($tmp);
}
}
}
if ($tmp) {
$data[$setting] = $tmp;
}
}
}
// Make sure that we have something to display
$accept = $data['title'] && trim($data['title']) != '' || $data['text'] && trim($data['text']) != '' || $data['image'];
if (!$accept) {
continue;
}
$items[] = $data;
if ($itemsCnt !== null) {
if (--$itemsCnt === 0) {
break;
}
}
}
$images = isset($config->content['image']) ? $config->content['image'] : true;
$moreLink = !isset($config->moreLink) || $config->moreLink ? $channel->getLink() : null;
$key = $touchDevice ? 'touch' : 'desktop';
$linkText = null;
if (isset($config->linkText[$key])) {
$linkText = $config->linkText[$key];
} else {
if (isset($config->linkText) && is_string($config->linkText)) {
$linkText = $config->linkText;
}
}
$feed = ['linkText' => $linkText, 'moreLink' => $moreLink, 'type' => $type, 'items' => $items, 'touchDevice' => $touchDevice, 'images' => $images];
if (isset($config->title)) {
if ($config->title == 'rss') {
$feed['title'] = $channel->getTitle();
} else {
$feed['translateTitle'] = $config->title;
}
}
if (isset($config->linkTarget)) {
$feed['linkTarget'] = $config->linkTarget;
}
$template = $type == 'list' ? 'list' : 'carousel';
$html = $this->getViewRenderer()->partial("ajax/feed-{$template}.phtml", $feed);
$settings = [];
$settings['type'] = $type;
if (isset($config->height)) {
$settings['height'] = $config->height;
}
if ($type == 'carousel' || $type == 'carousel-vertical') {
$settings['images'] = $images;
$settings['autoplay'] = isset($config->autoplay) ? $config->autoplay : false;
$settings['dots'] = isset($config->dots) ? $config->dots == true : true;
$breakPoints = ['desktop' => 4, 'desktop-small' => 3, 'tablet' => 2, 'mobile' => 1];
foreach ($breakPoints as $breakPoint => $default) {
$settings['slidesToShow'][$breakPoint] = isset($config->itemsPerPage[$breakPoint]) ? (int) $config->itemsPerPage[$breakPoint] : $default;
$settings['scrolledItems'][$breakPoint] = isset($config->scrolledItems[$breakPoint]) ? (int) $config->scrolledItems[$breakPoint] : $settings['slidesToShow'][$breakPoint];
}
if ($type == 'carousel') {
$settings['titlePosition'] = isset($config->titlePosition) ? $config->titlePosition : null;
}
}
$res = ['html' => $html, 'settings' => $settings];
return $this->output($res, self::STATUS_OK);
}
/**
* Import a feed by providing a URI
*
* @param string $uri
* The URI to the feed
* @param string $format
* The output format. Possible values xml or json (default).
* @return array
*/
public function feedRead($uri, $format = "json", $fixencoding = true)
{
$zuri = \Zend\Uri\UriFactory::factory($uri);
if (!$zuri->isValid()) {
$this->logger->logWarn("Invalid Uri ({$uri})");
return false;
}
$cache = \Zend\Cache\StorageFactory::factory(array('adapter' => array('name' => 'filesystem', 'options' => array('cache_dir' => __DIR__ . DIRECTORY_SEPARATOR . 'cache', 'ttl' => 3600)), 'plugins' => array(array('name' => 'serializer', 'options' => array()))));
\Zend\Feed\Reader\Reader::setCache($cache);
\Zend\Feed\Reader\Reader::useHttpConditionalGet();
// Change to firefox agent
$httpClient = \Zend\Feed\Reader\Reader::getHttpClient();
$httpClient->setOptions(array('useragent' => 'Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0', 'timeout' => 25));
\Zend\Feed\Reader\Reader::setHttpClient($httpClient);
// Import feed
try {
$feed = \Zend\Feed\Reader\Reader::import($uri);
$httpClient = \Zend\Feed\Reader\Reader::getHttpClient();
$httpClientResponse = $httpClient->getResponse();
$feedencoding = $feed->getEncoding();
if (!$feedencoding) {
$feedencoding = 'utf-8';
}
$feedResponse = array('feed' => $uri, 'statuscode' => $httpClientResponse->getStatusCode(), 'headers' => $httpClientResponse->getHeaders()->toArray(), 'encoding' => $feedencoding);
} catch (\Exception $e) {
$this->logger->logWarn("Zend feed reader cannot fetch: ({$uri}) because : " . $e->getMessage() . " trying casperjs");
$command = 'casperjs ' . __DIR__ . '/html.js --cookies-file=cookies.txt --url=' . escapeshellarg($uri);
exec($command, $response, $return_var);
$response = $this->strip_json($response);
if ($return_var !== 0 || !isset($response['content']) || empty($response['content'])) {
$this->logger->logWarn("Cannot fetch: ({$uri}) exit code {$return_var}", array('command' => $command, 'response' => $response));
$failed = new \Rss\Feed\Reader\ReadFailures($this->postgresql);
$failed->update_failure($uri, json_encode(array('zend' => $e->getMessage(), 'casper' => $response)));
return false;
}
try {
$feed = \Zend\Feed\Reader\Reader::importString($response['content']);
$feedencoding = $feed->getEncoding();
if (!$feedencoding) {
$feedencoding = 'utf-8';
}
$feedResponse = array('feed' => $uri, 'statuscode' => $response['http']['status'], 'headers' => $response['http'], 'encoding' => $feedencoding);
} catch (\Exception $e) {
$this->logger->logWarn("Cannot parse feed content from ({$uri}) because " . $e->getMessage());
// $failed = new \Rss\Feed\Reader\ReadFailures ( $this->postgresql );
// $failed->update_failure ( $uri );
return false;
}
}
// Fix relative links
$newdata = $this->fix_links($feed, $uri);
// Fix encoding errors
if ($fixencoding) {
$newdata = $this->fix_encoding($newdata);
}
// Return XML
if ($format === "xml") {
$feedResponse['data'] = $newdata;
return $feedResponse;
}
// Reload fixed data
try {
$feed = \Zend\Feed\Reader\Reader::importString($newdata);
} catch (\Exception $e) {
$this->logger->logWarn("Cannot parse corrected feed content from ({$uri}) because " . $e->getMessage());
return false;
}
$data = array('title' => $feed->getTitle(), 'link' => $feed->getLink(), 'date' => $this->get_feed_date($feed), 'description' => $feed->getDescription(), 'language' => $feed->getLanguage(), 'entries' => array());
foreach ($feed as $entry) {
if (is_object($entry)) {
$DateCreated = $entry->getDateCreated();
if (is_object($DateCreated)) {
$DateCreated = $DateCreated->getTimestamp();
} else {
$DateCreated = $data['date'];
}
$DateModified = $entry->getDateModified();
if (is_object($DateModified)) {
$DateModified = $DateModified->getTimestamp();
} else {
$DateModified = $data['date'];
}
if (empty($DateModified)) {
$DateModified = time();
}
if (empty($DateCreated)) {
$DateCreated = time();
}
$description = $entry->getDescription();
$content = $entry->getContent();
if (empty($description)) {
$description = $content;
}
if (empty($content)) {
$content = $description;
}
$edata = array('title' => $entry->getTitle(), 'description' => $description, 'dateCreated' => $DateCreated, 'dateModified' => $DateModified, 'link' => $entry->getLink(), 'content' => $content, 'feed' => $uri);
if ($this->check_missing($edata)) {
$data['entries'][] = $edata;
} else {
$this->logger->logTrace("Missing data from feed {$uri} " . json_encode($edata, JSON_UNESCAPED_UNICODE));
}
}
}
// Return array
$feedResponse['data'] = $data;
return $feedResponse;
}
