/**
* {@inheritdoc}
*/
public function parse(FeedInterface $feed, FetcherResultInterface $fetcher_result, StateInterface $state)
{
$result = new ParserResult();
Reader::setExtensionManager(\Drupal::service('feed.bridge.reader'));
Reader::registerExtension('GeoRSS');
$raw = $fetcher_result->getRaw();
if (!strlen(trim($raw))) {
throw new EmptyFeedException();
}
try {
$channel = Reader::importString($raw);
} catch (ExceptionInterface $e) {
$args = ['%site' => $feed->label(), '%error' => trim($e->getMessage())];
throw new \RuntimeException($this->t('The feed from %site seems to be broken because of error "%error".', $args));
}
foreach ($channel as $delta => $entry) {
$item = new SyndicationItem();
// Move the values to an array as expected by processors.
$item->set('title', $entry->getTitle())->set('guid', $entry->getId())->set('url', $entry->getLink())->set('guid', $entry->getId())->set('url', $entry->getLink())->set('description', $entry->getDescription())->set('tags', $entry->getCategories()->getValues())->set('feed_title', $channel->getTitle())->set('feed_description', $channel->getDescription())->set('feed_url', $channel->getLink());
if ($image = $channel->getImage()) {
$item->set('feed_image_uri', $image['uri']);
}
if ($enclosure = $entry->getEnclosure()) {
$item->set('enclosures', [rawurldecode($enclosure->url)]);
}
if ($author = $entry->getAuthor()) {
$author += ['name' => '', 'email' => ''];
$item->set('author_name', $author['name'])->set('author_email', $author['email']);
}
if ($date = $entry->getDateModified()) {
$item->set('timestamp', $date->getTimestamp());
}
if ($point = $entry->getGeoPoint()) {
$item->set('georss_lat', $point['lat'])->set('georss_lon', $point['lon']);
}
$result->addItem($item);
}
return $result;
}
/**
* {@inheritdoc}
*/
public function parse(FeedInterface $feed)
{
// Set our bridge extension manager to Zend Feed.
Reader::setExtensionManager(\Drupal::service('feed.bridge.reader'));
try {
$channel = Reader::importString($feed->source_string);
} catch (ExceptionInterface $e) {
watchdog_exception('aggregator', $e);
drupal_set_message(t('The feed from %site seems to be broken because of error "%error".', array('%site' => $feed->label(), '%error' => $e->getMessage())), 'error');
return FALSE;
}
$feed->setWebsiteUrl($channel->getLink());
$feed->setDescription($channel->getDescription());
if ($image = $channel->getImage()) {
$feed->setImage($image['uri']);
}
// Initialize items array.
$feed->items = array();
foreach ($channel as $item) {
// Reset the parsed item.
$parsed_item = array();
// Move the values to an array as expected by processors.
$parsed_item['title'] = $item->getTitle();
$parsed_item['guid'] = $item->getId();
$parsed_item['link'] = $item->getLink();
$parsed_item['description'] = $item->getDescription();
$parsed_item['author'] = '';
if ($author = $item->getAuthor()) {
$parsed_item['author'] = $author['name'];
}
$parsed_item['timestamp'] = '';
if ($date = $item->getDateModified()) {
$parsed_item['timestamp'] = $date->getTimestamp();
}
// Store on $feed object. This is where processors will look for parsed items.
$feed->items[] = $parsed_item;
}
return TRUE;
}
/**
* {@inheritdoc}
*/
public function parse(FeedInterface $feed, FetcherResultInterface $fetcher_result)
{
$result = new ParserResult();
Reader::setExtensionManager(\Drupal::service('feed.bridge.reader'));
try {
$channel = Reader::importString($fetcher_result->getRaw());
} catch (ExceptionInterface $e) {
watchdog_exception('feeds', $e);
drupal_set_message($this->t('The feed from %site seems to be broken because of error "%error".', array('%site' => $feed->label(), '%error' => $e->getMessage())), 'error');
return $result;
}
$result->title = $channel->getTitle();
$result->description = $channel->getDescription();
$result->link = $channel->getLink();
foreach ($channel as $item) {
// Reset the parsed item.
$parsed_item = array();
// Move the values to an array as expected by processors.
$parsed_item['title'] = $item->getTitle();
$parsed_item['guid'] = $item->getId();
$parsed_item['url'] = $item->getLink();
$parsed_item['description'] = $item->getDescription();
if ($enclosure = $item->getEnclosure()) {
$parsed_item['enclosures'][] = urldecode($enclosure->url);
}
if ($author = $item->getAuthor()) {
$parsed_item['author_name'] = $author['name'];
}
if ($date = $item->getDateModified()) {
$parsed_item['timestamp'] = $date->getTimestamp();
}
$parsed_item['tags'] = $item->getCategories()->getValues();
$result->items[] = $parsed_item;
}
return $result;
}
/**
* Set content for worker
*
* @param string $content
* @return Worker
*/
public function setContent($content)
{
if ($content == null) {
$this->errors = 'Connection timeout';
} else {
$content = trim($content);
if (empty($content)) {
$this->errors = 'Source content is empty.';
} else {
try {
Reader::registerExtension('Syndication');
$this->feed = Reader::importString($content);
$this->errors = '';
} catch (InvalidArgumentException $ex) {
$this->errors = $ex->getMessage();
$this->feed = null;
} catch (RuntimeException $ex) {
$this->errors = $ex->getMessage();
$this->feed = null;
}
}
}
return $this;
}
public function testXxePreventionOnFeedParsing()
{
$this->setExpectedException('Zend\\Feed\\Reader\\Exception\\InvalidArgumentException');
$string = file_get_contents($this->feedSamplePath . '/Reader/xxe-atom10.xml');
$string = str_replace('XXE_URI', $this->feedSamplePath . '/Reader/xxe-info.txt', $string);
$feed = Reader\Reader::importString($string);
//$this->assertEquals('info:', $feed->getTitle());
}
public function testGetsCategoriesFromAtom03_None()
{
$feed = Reader\Reader::importString(file_get_contents($this->_feedSamplePath . '/category/plain/none/atom03.xml'));
$entry = $feed->current();
$this->assertEquals(array(), (array) $entry->getCategories());
$this->assertEquals(array(), array_values($entry->getCategories()->getValues()));
}
/**
* When passing a newly created DOMElement without any DOMDocument assigned
*/
public function testFeedEntryCanSetAnyType()
{
$feed = Reader\Reader::importString(file_get_contents($this->_feedSamplePath . '/atom.xml'));
$entry = $feed->current();
$domElement = new \DOMElement($entry->getElement()->tagName);
$stub = $this->getMockForAbstractClass('Zend\\Feed\\Reader\\Entry\\AbstractEntry', array($domElement, $entry->getId()));
$this->assertEquals($stub->getType(), Reader\Reader::TYPE_ANY);
}
private function getFeedFromLocalFile(string $file) : FeedInterface
{
return FeedReader::importString(file_get_contents($file));
}
public function testGetsEntryEncoding()
{
$feed = Reader\Reader::importString(file_get_contents($this->feedSamplePath));
$entry = $feed->current();
$this->assertEquals('UTF-8', $entry->getEncoding());
}
public function testGetEncoding()
{
$feed = Reader\Reader::importString(file_get_contents(__DIR__ . '/Reader/Entry/_files/Atom/title/plain/atom10.xml'));
$this->assertEquals('utf-8', $feed->getEncoding());
$this->assertEquals('utf-8', $feed->current()->getEncoding());
}
public function testCommentFeedLinksRendered()
{
$renderer = new Renderer\Feed\Atom($this->_validWriter);
$this->_validEntry->setCommentFeedLinks(array(array('uri' => 'http://www.example.com/atom/id/1', 'type' => 'atom'), array('uri' => 'http://www.example.com/rss/id/1', 'type' => 'rss')));
$feed = Reader\Reader::importString($renderer->render()->saveXml());
$entry = $feed->current();
// Skipped over due to ZFR bug (detects Atom in error when RSS requested)
//$this->assertEquals('http://www.example.com/rss/id/1', $entry->getCommentFeedLink('rss'));
$this->assertEquals('http://www.example.com/atom/id/1', $entry->getCommentFeedLink('atom'));
}
public function testGetsEnclosure()
{
$feed = Reader\Reader::importString(
file_get_contents($this->_feedSamplePath)
);
$entry = $feed->current();
$expected = new \stdClass();
$expected->url = 'http://example.com/podcasts/everything/AllAboutEverythingEpisode3.m4a';
$expected->length = '8727310';
$expected->type = 'audio/x-m4a';
$this->assertEquals($expected, $entry->getEnclosure());
}
/**
* Processes content from successfully polled feeds.
*
* @param string $url URL of the polled feed
* @param string $data Content received from the feed poll
*/
public function processFeed($url, $data)
{
$logger = $this->getLogger();
$logger->info('Processing feed', array('url' => $url));
$logger->debug('Received feed data', array('data' => $data));
try {
$new = iterator_to_array(FeedReader::importString($data));
$old = isset($this->cache[$url]) ? $this->cache[$url] : array();
$diff = $this->getNewFeedItems($new, $old);
if ($old) {
$this->syndicateFeedItems($diff);
}
$this->cache[$url] = $new;
} catch (\Exception $e) {
$logger->warning('Failed to process feed', array('url' => $url, 'data' => $data, 'error' => $e));
}
$this->queuePoll($url);
}
/**
* Test feed generation
*
* @return void
*/
public function testRSS()
{
// Set up a request -- we'll sort by title to ensure a predictable order
// for the result list (relevance or last_indexed may lead to unstable test
// cases).
$request = new \Zend\Stdlib\Parameters();
$request->set('lookfor', 'id:testbug2 OR id:testsample1');
$request->set('skip_rss_sort', 1);
$request->set('sort', 'title');
$request->set('view', 'rss');
$results = $this->getServiceManager()->get('VuFind\\SearchResultsPluginManager')->get('Solr');
$results->getParams()->initFromRequest($request);
$helper = new ResultFeed();
$helper->setView($this->getPhpRenderer($this->getPlugins()));
$feed = $helper->__invoke($results, '/test/path');
$this->assertTrue(is_object($feed));
$rss = $feed->export('rss');
// Make sure it's really an RSS feed:
$this->assertTrue(strstr($rss, '<rss') !== false);
// Make sure custom Dublin Core elements are present:
$this->assertTrue(strstr($rss, 'dc:format') !== false);
// Now re-parse it and check for some expected values:
$parsedFeed = \Zend\Feed\Reader\Reader::importString($rss);
$this->assertEquals('Showing 1-2 of 2', $parsedFeed->getDescription());
$items = [];
$i = 0;
foreach ($parsedFeed as $item) {
$items[$i++] = $item;
}
$this->assertEquals('Journal of rational emotive therapy : ' . 'the journal of the Institute for Rational-Emotive Therapy.', $items[1]->getTitle());
}
public function testXxePreventionOnFeedParsing()
{
$string = file_get_contents($this->_feedSamplePath . '/Reader/xxe-atom10.xml');
$string = str_replace('XXE_URI', $this->_feedSamplePath . '/Reader/xxe-info.txt', $string);
$feed = Reader\Reader::importString($string);
$this->assertEquals('info:', $feed->getTitle());
}
/**
* Performs a Twitter search query.
*
* @param string $query (optional)
* @param array|\Traversable|SearchOptions $options (optional)
* @throws Exception\InvalidArgumentException If query is not defined neither $query nor $options or
* If query is not a string.
* @throws Http\Client\Exception\ExceptionInterface
* @return mixed
*/
public function execute($query = null, $options = null)
{
if (!$options) {
$options = $this->getOptions();
} elseif (!$options instanceof SearchOptions) {
$options = new SearchOptions($options);
}
if (is_string($query)) {
$options->setQuery($query);
} else {
if ($query) {
throw new Exception\InvalidArgumentException('query must be a string');
}
if (!$options->getQuery()) {
throw new Exception\InvalidArgumentException('No query defined');
}
}
$response = $this->restGet('/search.' . $this->responseType, $options->toArray());
switch ($this->responseType) {
case 'json':
return Json\Json::decode($response->getBody(), Json\Json::TYPE_ARRAY);
break;
case 'atom':
return Feed\Reader\Reader::importString($response->getBody());
break;
}
return null;
}
public function testImportStringMethodThrowProperExceptionOnEmptyString()
{
$this->setExpectedException('Zend\\Feed\\Reader\\Exception\\InvalidArgumentException');
$string = ' ';
$feed = Reader\Reader::importString($string);
}
/**
* Get category data
*/
public function testGetsCategoriesFromAtom10()
{
$feed = Reader\Reader::importString(file_get_contents($this->feedSamplePath . '/category/atom10.xml'));
$source = $feed->current()->getSource();
$this->assertEquals($this->expectedCats, (array) $source->getCategories());
$this->assertEquals(array('topic1', 'Cat & Dog'), array_values($source->getCategories()->getValues()));
}
public function testImageCanBeSetWithOnlyRequiredElements()
{
$this->_validWriter->setImage(array('uri' => 'http://www.example.com/logo.gif', 'link' => 'http://www.example.com', 'title' => 'Image ALT'));
$rssFeed = new Renderer\Feed\Rss($this->_validWriter);
$rssFeed->render();
$feed = Reader\Reader::importString($rssFeed->saveXml());
$expected = array('uri' => 'http://www.example.com/logo.gif', 'link' => 'http://www.example.com', 'title' => 'Image ALT');
$this->assertEquals($expected, $feed->getImage());
}
/**
* @group ZF-8213
*/
public function testReturnsEncodingOfFeedAsUtf8IfUndefined()
{
$feed = Reader\Reader::importString(file_get_contents($this->_feedSamplePath . '/atom_noencodingdefined.xml'));
$this->assertEquals('UTF-8', $feed->getEncoding());
}
public function testFeedSetDateCreated()
{
$this->validWriter->setDateCreated(1234567890);
$rssFeed = new Renderer\Feed\Rss($this->validWriter);
$rssFeed->render();
$feed = Reader\Reader::importString($rssFeed->saveXml());
$myDate = new DateTime('@' . 1234567890);
$this->assertEquals($myDate, $feed->getDateCreated());
}
/**
* Get category data
* @group ZFR002
*/
public function testGetsCategoriesFromAtom10()
{
$entry = Reader\Reader::importString(file_get_contents($this->_feedSamplePath . '/category/atom10.xml'));
$this->assertEquals($this->_expectedCats, (array) $entry->getCategories());
$this->assertEquals(array('topic1', 'Cat & Dog'), array_values($entry->getCategories()->getValues()));
}
/**
* @group ZFWCHARDATA01
*/
public function testCategoriesCharDataEncoding()
{
$this->_validEntry->addCategories(array(array('term' => '<>&\'"áéíóú', 'label' => 'Cats & Dogs', 'scheme' => 'http://example.com/schema1'), array('term' => 'cat_dog2')));
$renderer = new Renderer\Feed\Rss($this->_validWriter);
$feed = Reader\Reader::importString($renderer->render()->saveXml());
$entry = $feed->current();
$expected = array(array('term' => '<>&\'"áéíóú', 'label' => '<>&\'"áéíóú', 'scheme' => 'http://example.com/schema1'), array('term' => 'cat_dog2', 'label' => 'cat_dog2', 'scheme' => null));
$this->assertEquals($expected, (array) $entry->getCategories());
}
public function testImageCanBeSet()
{
$this->validWriter->setImage(array('uri' => 'http://www.example.com/logo.gif'));
$atomFeed = new Renderer\Feed\Atom($this->validWriter);
$atomFeed->render();
$feed = Reader\Reader::importString($atomFeed->saveXml());
$expected = array('uri' => 'http://www.example.com/logo.gif');
$this->assertEquals($expected, $feed->getImage());
}
/**
* Import a feed by providing a URI
*
* @param string $uri
* The URI to the feed
* @param string $format
* The output format. Possible values xml or json (default).
* @return array
*/
public function feedRead($uri, $format = "json", $fixencoding = true)
{
$zuri = \Zend\Uri\UriFactory::factory($uri);
if (!$zuri->isValid()) {
$this->logger->logWarn("Invalid Uri ({$uri})");
return false;
}
$cache = \Zend\Cache\StorageFactory::factory(array('adapter' => array('name' => 'filesystem', 'options' => array('cache_dir' => __DIR__ . DIRECTORY_SEPARATOR . 'cache', 'ttl' => 3600)), 'plugins' => array(array('name' => 'serializer', 'options' => array()))));
\Zend\Feed\Reader\Reader::setCache($cache);
\Zend\Feed\Reader\Reader::useHttpConditionalGet();
// Change to firefox agent
$httpClient = \Zend\Feed\Reader\Reader::getHttpClient();
$httpClient->setOptions(array('useragent' => 'Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0', 'timeout' => 25));
\Zend\Feed\Reader\Reader::setHttpClient($httpClient);
// Import feed
try {
$feed = \Zend\Feed\Reader\Reader::import($uri);
$httpClient = \Zend\Feed\Reader\Reader::getHttpClient();
$httpClientResponse = $httpClient->getResponse();
$feedencoding = $feed->getEncoding();
if (!$feedencoding) {
$feedencoding = 'utf-8';
}
$feedResponse = array('feed' => $uri, 'statuscode' => $httpClientResponse->getStatusCode(), 'headers' => $httpClientResponse->getHeaders()->toArray(), 'encoding' => $feedencoding);
} catch (\Exception $e) {
$this->logger->logWarn("Zend feed reader cannot fetch: ({$uri}) because : " . $e->getMessage() . " trying casperjs");
$command = 'casperjs ' . __DIR__ . '/html.js --cookies-file=cookies.txt --url=' . escapeshellarg($uri);
exec($command, $response, $return_var);
$response = $this->strip_json($response);
if ($return_var !== 0 || !isset($response['content']) || empty($response['content'])) {
$this->logger->logWarn("Cannot fetch: ({$uri}) exit code {$return_var}", array('command' => $command, 'response' => $response));
$failed = new \Rss\Feed\Reader\ReadFailures($this->postgresql);
$failed->update_failure($uri, json_encode(array('zend' => $e->getMessage(), 'casper' => $response)));
return false;
}
try {
$feed = \Zend\Feed\Reader\Reader::importString($response['content']);
$feedencoding = $feed->getEncoding();
if (!$feedencoding) {
$feedencoding = 'utf-8';
}
$feedResponse = array('feed' => $uri, 'statuscode' => $response['http']['status'], 'headers' => $response['http'], 'encoding' => $feedencoding);
} catch (\Exception $e) {
$this->logger->logWarn("Cannot parse feed content from ({$uri}) because " . $e->getMessage());
// $failed = new \Rss\Feed\Reader\ReadFailures ( $this->postgresql );
// $failed->update_failure ( $uri );
return false;
}
}
// Fix relative links
$newdata = $this->fix_links($feed, $uri);
// Fix encoding errors
if ($fixencoding) {
$newdata = $this->fix_encoding($newdata);
}
// Return XML
if ($format === "xml") {
$feedResponse['data'] = $newdata;
return $feedResponse;
}
// Reload fixed data
try {
$feed = \Zend\Feed\Reader\Reader::importString($newdata);
} catch (\Exception $e) {
$this->logger->logWarn("Cannot parse corrected feed content from ({$uri}) because " . $e->getMessage());
return false;
}
$data = array('title' => $feed->getTitle(), 'link' => $feed->getLink(), 'date' => $this->get_feed_date($feed), 'description' => $feed->getDescription(), 'language' => $feed->getLanguage(), 'entries' => array());
foreach ($feed as $entry) {
if (is_object($entry)) {
$DateCreated = $entry->getDateCreated();
if (is_object($DateCreated)) {
$DateCreated = $DateCreated->getTimestamp();
} else {
$DateCreated = $data['date'];
}
$DateModified = $entry->getDateModified();
if (is_object($DateModified)) {
$DateModified = $DateModified->getTimestamp();
} else {
$DateModified = $data['date'];
}
if (empty($DateModified)) {
$DateModified = time();
}
if (empty($DateCreated)) {
$DateCreated = time();
}
$description = $entry->getDescription();
$content = $entry->getContent();
if (empty($description)) {
$description = $content;
}
if (empty($content)) {
$content = $description;
}
$edata = array('title' => $entry->getTitle(), 'description' => $description, 'dateCreated' => $DateCreated, 'dateModified' => $DateModified, 'link' => $entry->getLink(), 'content' => $content, 'feed' => $uri);
if ($this->check_missing($edata)) {
$data['entries'][] = $edata;
} else {
$this->logger->logTrace("Missing data from feed {$uri} " . json_encode($edata, JSON_UNESCAPED_UNICODE));
}
}
}
// Return array
$feedResponse['data'] = $data;
return $feedResponse;
}
public function testGetsImageFromAtom10_None()
{
$feed = Reader\Reader::importString(file_get_contents($this->feedSamplePath . '/image/plain/none/atom10.xml'));
$this->assertEquals(null, $feed->getImage());
}
/**
* Performs a Twitter search query.
*
* @throws Http\Client\Exception
* @return mixed
*/
public function execute($query, array $params = array())
{
$_query = array();
$_query['q'] = $query;
foreach ($params as $key => $param) {
switch ($key) {
case 'geocode':
case 'lang':
case 'since_id':
$_query[$key] = $param;
break;
case 'rpp':
$_query[$key] = intval($param) > 100 ? 100 : intval($param);
break;
case 'page':
$_query[$key] = intval($param);
break;
case 'show_user':
$_query[$key] = 'true';
}
}
$response = $this->restGet('/search.' . $this->responseType, $_query);
switch ($this->responseType) {
case 'json':
return Json\Json::decode($response->getBody(), Json\Json::TYPE_ARRAY);
break;
case 'atom':
return Feed\Reader\Reader::importString($response->getBody());
break;
}
return;
}
