/** * {@inheritdoc} */ public function parse(FeedInterface $feed, FetcherResultInterface $fetcher_result, StateInterface $state) { $result = new ParserResult(); Reader::setExtensionManager(\Drupal::service('feed.bridge.reader')); Reader::registerExtension('GeoRSS'); $raw = $fetcher_result->getRaw(); if (!strlen(trim($raw))) { throw new EmptyFeedException(); } try { $channel = Reader::importString($raw); } catch (ExceptionInterface $e) { $args = ['%site' => $feed->label(), '%error' => trim($e->getMessage())]; throw new \RuntimeException($this->t('The feed from %site seems to be broken because of error "%error".', $args)); } foreach ($channel as $delta => $entry) { $item = new SyndicationItem(); // Move the values to an array as expected by processors. $item->set('title', $entry->getTitle())->set('guid', $entry->getId())->set('url', $entry->getLink())->set('guid', $entry->getId())->set('url', $entry->getLink())->set('description', $entry->getDescription())->set('tags', $entry->getCategories()->getValues())->set('feed_title', $channel->getTitle())->set('feed_description', $channel->getDescription())->set('feed_url', $channel->getLink()); if ($image = $channel->getImage()) { $item->set('feed_image_uri', $image['uri']); } if ($enclosure = $entry->getEnclosure()) { $item->set('enclosures', [rawurldecode($enclosure->url)]); } if ($author = $entry->getAuthor()) { $author += ['name' => '', 'email' => '']; $item->set('author_name', $author['name'])->set('author_email', $author['email']); } if ($date = $entry->getDateModified()) { $item->set('timestamp', $date->getTimestamp()); } if ($point = $entry->getGeoPoint()) { $item->set('georss_lat', $point['lat'])->set('georss_lon', $point['lon']); } $result->addItem($item); } return $result; }
/** * {@inheritdoc} */ public function parse(FeedInterface $feed) { // Set our bridge extension manager to Zend Feed. Reader::setExtensionManager(\Drupal::service('feed.bridge.reader')); try { $channel = Reader::importString($feed->source_string); } catch (ExceptionInterface $e) { watchdog_exception('aggregator', $e); drupal_set_message(t('The feed from %site seems to be broken because of error "%error".', array('%site' => $feed->label(), '%error' => $e->getMessage())), 'error'); return FALSE; } $feed->setWebsiteUrl($channel->getLink()); $feed->setDescription($channel->getDescription()); if ($image = $channel->getImage()) { $feed->setImage($image['uri']); } // Initialize items array. $feed->items = array(); foreach ($channel as $item) { // Reset the parsed item. $parsed_item = array(); // Move the values to an array as expected by processors. $parsed_item['title'] = $item->getTitle(); $parsed_item['guid'] = $item->getId(); $parsed_item['link'] = $item->getLink(); $parsed_item['description'] = $item->getDescription(); $parsed_item['author'] = ''; if ($author = $item->getAuthor()) { $parsed_item['author'] = $author['name']; } $parsed_item['timestamp'] = ''; if ($date = $item->getDateModified()) { $parsed_item['timestamp'] = $date->getTimestamp(); } // Store on $feed object. This is where processors will look for parsed items. $feed->items[] = $parsed_item; } return TRUE; }
/** * {@inheritdoc} */ public function parse(FeedInterface $feed, FetcherResultInterface $fetcher_result) { $result = new ParserResult(); Reader::setExtensionManager(\Drupal::service('feed.bridge.reader')); try { $channel = Reader::importString($fetcher_result->getRaw()); } catch (ExceptionInterface $e) { watchdog_exception('feeds', $e); drupal_set_message($this->t('The feed from %site seems to be broken because of error "%error".', array('%site' => $feed->label(), '%error' => $e->getMessage())), 'error'); return $result; } $result->title = $channel->getTitle(); $result->description = $channel->getDescription(); $result->link = $channel->getLink(); foreach ($channel as $item) { // Reset the parsed item. $parsed_item = array(); // Move the values to an array as expected by processors. $parsed_item['title'] = $item->getTitle(); $parsed_item['guid'] = $item->getId(); $parsed_item['url'] = $item->getLink(); $parsed_item['description'] = $item->getDescription(); if ($enclosure = $item->getEnclosure()) { $parsed_item['enclosures'][] = urldecode($enclosure->url); } if ($author = $item->getAuthor()) { $parsed_item['author_name'] = $author['name']; } if ($date = $item->getDateModified()) { $parsed_item['timestamp'] = $date->getTimestamp(); } $parsed_item['tags'] = $item->getCategories()->getValues(); $result->items[] = $parsed_item; } return $result; }
/** * Set content for worker * * @param string $content * @return Worker */ public function setContent($content) { if ($content == null) { $this->errors = 'Connection timeout'; } else { $content = trim($content); if (empty($content)) { $this->errors = 'Source content is empty.'; } else { try { Reader::registerExtension('Syndication'); $this->feed = Reader::importString($content); $this->errors = ''; } catch (InvalidArgumentException $ex) { $this->errors = $ex->getMessage(); $this->feed = null; } catch (RuntimeException $ex) { $this->errors = $ex->getMessage(); $this->feed = null; } } } return $this; }
public function testXxePreventionOnFeedParsing() { $this->setExpectedException('Zend\\Feed\\Reader\\Exception\\InvalidArgumentException'); $string = file_get_contents($this->feedSamplePath . '/Reader/xxe-atom10.xml'); $string = str_replace('XXE_URI', $this->feedSamplePath . '/Reader/xxe-info.txt', $string); $feed = Reader\Reader::importString($string); //$this->assertEquals('info:', $feed->getTitle()); }
public function testGetsCategoriesFromAtom03_None() { $feed = Reader\Reader::importString(file_get_contents($this->_feedSamplePath . '/category/plain/none/atom03.xml')); $entry = $feed->current(); $this->assertEquals(array(), (array) $entry->getCategories()); $this->assertEquals(array(), array_values($entry->getCategories()->getValues())); }
/** * When passing a newly created DOMElement without any DOMDocument assigned */ public function testFeedEntryCanSetAnyType() { $feed = Reader\Reader::importString(file_get_contents($this->_feedSamplePath . '/atom.xml')); $entry = $feed->current(); $domElement = new \DOMElement($entry->getElement()->tagName); $stub = $this->getMockForAbstractClass('Zend\\Feed\\Reader\\Entry\\AbstractEntry', array($domElement, $entry->getId())); $this->assertEquals($stub->getType(), Reader\Reader::TYPE_ANY); }
private function getFeedFromLocalFile(string $file) : FeedInterface { return FeedReader::importString(file_get_contents($file)); }
public function testGetsEntryEncoding() { $feed = Reader\Reader::importString(file_get_contents($this->feedSamplePath)); $entry = $feed->current(); $this->assertEquals('UTF-8', $entry->getEncoding()); }
public function testGetEncoding() { $feed = Reader\Reader::importString(file_get_contents(__DIR__ . '/Reader/Entry/_files/Atom/title/plain/atom10.xml')); $this->assertEquals('utf-8', $feed->getEncoding()); $this->assertEquals('utf-8', $feed->current()->getEncoding()); }
public function testCommentFeedLinksRendered() { $renderer = new Renderer\Feed\Atom($this->_validWriter); $this->_validEntry->setCommentFeedLinks(array(array('uri' => 'http://www.example.com/atom/id/1', 'type' => 'atom'), array('uri' => 'http://www.example.com/rss/id/1', 'type' => 'rss'))); $feed = Reader\Reader::importString($renderer->render()->saveXml()); $entry = $feed->current(); // Skipped over due to ZFR bug (detects Atom in error when RSS requested) //$this->assertEquals('http://www.example.com/rss/id/1', $entry->getCommentFeedLink('rss')); $this->assertEquals('http://www.example.com/atom/id/1', $entry->getCommentFeedLink('atom')); }
public function testGetsEnclosure() { $feed = Reader\Reader::importString( file_get_contents($this->_feedSamplePath) ); $entry = $feed->current(); $expected = new \stdClass(); $expected->url = 'http://example.com/podcasts/everything/AllAboutEverythingEpisode3.m4a'; $expected->length = '8727310'; $expected->type = 'audio/x-m4a'; $this->assertEquals($expected, $entry->getEnclosure()); }
/** * Processes content from successfully polled feeds. * * @param string $url URL of the polled feed * @param string $data Content received from the feed poll */ public function processFeed($url, $data) { $logger = $this->getLogger(); $logger->info('Processing feed', array('url' => $url)); $logger->debug('Received feed data', array('data' => $data)); try { $new = iterator_to_array(FeedReader::importString($data)); $old = isset($this->cache[$url]) ? $this->cache[$url] : array(); $diff = $this->getNewFeedItems($new, $old); if ($old) { $this->syndicateFeedItems($diff); } $this->cache[$url] = $new; } catch (\Exception $e) { $logger->warning('Failed to process feed', array('url' => $url, 'data' => $data, 'error' => $e)); } $this->queuePoll($url); }
/** * Test feed generation * * @return void */ public function testRSS() { // Set up a request -- we'll sort by title to ensure a predictable order // for the result list (relevance or last_indexed may lead to unstable test // cases). $request = new \Zend\Stdlib\Parameters(); $request->set('lookfor', 'id:testbug2 OR id:testsample1'); $request->set('skip_rss_sort', 1); $request->set('sort', 'title'); $request->set('view', 'rss'); $results = $this->getServiceManager()->get('VuFind\\SearchResultsPluginManager')->get('Solr'); $results->getParams()->initFromRequest($request); $helper = new ResultFeed(); $helper->setView($this->getPhpRenderer($this->getPlugins())); $feed = $helper->__invoke($results, '/test/path'); $this->assertTrue(is_object($feed)); $rss = $feed->export('rss'); // Make sure it's really an RSS feed: $this->assertTrue(strstr($rss, '<rss') !== false); // Make sure custom Dublin Core elements are present: $this->assertTrue(strstr($rss, 'dc:format') !== false); // Now re-parse it and check for some expected values: $parsedFeed = \Zend\Feed\Reader\Reader::importString($rss); $this->assertEquals('Showing 1-2 of 2', $parsedFeed->getDescription()); $items = []; $i = 0; foreach ($parsedFeed as $item) { $items[$i++] = $item; } $this->assertEquals('Journal of rational emotive therapy : ' . 'the journal of the Institute for Rational-Emotive Therapy.', $items[1]->getTitle()); }
public function testXxePreventionOnFeedParsing() { $string = file_get_contents($this->_feedSamplePath . '/Reader/xxe-atom10.xml'); $string = str_replace('XXE_URI', $this->_feedSamplePath . '/Reader/xxe-info.txt', $string); $feed = Reader\Reader::importString($string); $this->assertEquals('info:', $feed->getTitle()); }
/** * Performs a Twitter search query. * * @param string $query (optional) * @param array|\Traversable|SearchOptions $options (optional) * @throws Exception\InvalidArgumentException If query is not defined neither $query nor $options or * If query is not a string. * @throws Http\Client\Exception\ExceptionInterface * @return mixed */ public function execute($query = null, $options = null) { if (!$options) { $options = $this->getOptions(); } elseif (!$options instanceof SearchOptions) { $options = new SearchOptions($options); } if (is_string($query)) { $options->setQuery($query); } else { if ($query) { throw new Exception\InvalidArgumentException('query must be a string'); } if (!$options->getQuery()) { throw new Exception\InvalidArgumentException('No query defined'); } } $response = $this->restGet('/search.' . $this->responseType, $options->toArray()); switch ($this->responseType) { case 'json': return Json\Json::decode($response->getBody(), Json\Json::TYPE_ARRAY); break; case 'atom': return Feed\Reader\Reader::importString($response->getBody()); break; } return null; }
public function testImportStringMethodThrowProperExceptionOnEmptyString() { $this->setExpectedException('Zend\\Feed\\Reader\\Exception\\InvalidArgumentException'); $string = ' '; $feed = Reader\Reader::importString($string); }
/** * Get category data */ public function testGetsCategoriesFromAtom10() { $feed = Reader\Reader::importString(file_get_contents($this->feedSamplePath . '/category/atom10.xml')); $source = $feed->current()->getSource(); $this->assertEquals($this->expectedCats, (array) $source->getCategories()); $this->assertEquals(array('topic1', 'Cat & Dog'), array_values($source->getCategories()->getValues())); }
public function testImageCanBeSetWithOnlyRequiredElements() { $this->_validWriter->setImage(array('uri' => 'http://www.example.com/logo.gif', 'link' => 'http://www.example.com', 'title' => 'Image ALT')); $rssFeed = new Renderer\Feed\Rss($this->_validWriter); $rssFeed->render(); $feed = Reader\Reader::importString($rssFeed->saveXml()); $expected = array('uri' => 'http://www.example.com/logo.gif', 'link' => 'http://www.example.com', 'title' => 'Image ALT'); $this->assertEquals($expected, $feed->getImage()); }
/** * @group ZF-8213 */ public function testReturnsEncodingOfFeedAsUtf8IfUndefined() { $feed = Reader\Reader::importString(file_get_contents($this->_feedSamplePath . '/atom_noencodingdefined.xml')); $this->assertEquals('UTF-8', $feed->getEncoding()); }
public function testFeedSetDateCreated() { $this->validWriter->setDateCreated(1234567890); $rssFeed = new Renderer\Feed\Rss($this->validWriter); $rssFeed->render(); $feed = Reader\Reader::importString($rssFeed->saveXml()); $myDate = new DateTime('@' . 1234567890); $this->assertEquals($myDate, $feed->getDateCreated()); }
/** * Get category data * @group ZFR002 */ public function testGetsCategoriesFromAtom10() { $entry = Reader\Reader::importString(file_get_contents($this->_feedSamplePath . '/category/atom10.xml')); $this->assertEquals($this->_expectedCats, (array) $entry->getCategories()); $this->assertEquals(array('topic1', 'Cat & Dog'), array_values($entry->getCategories()->getValues())); }
/** * @group ZFWCHARDATA01 */ public function testCategoriesCharDataEncoding() { $this->_validEntry->addCategories(array(array('term' => '<>&\'"áéíóú', 'label' => 'Cats & Dogs', 'scheme' => 'http://example.com/schema1'), array('term' => 'cat_dog2'))); $renderer = new Renderer\Feed\Rss($this->_validWriter); $feed = Reader\Reader::importString($renderer->render()->saveXml()); $entry = $feed->current(); $expected = array(array('term' => '<>&\'"áéíóú', 'label' => '<>&\'"áéíóú', 'scheme' => 'http://example.com/schema1'), array('term' => 'cat_dog2', 'label' => 'cat_dog2', 'scheme' => null)); $this->assertEquals($expected, (array) $entry->getCategories()); }
public function testImageCanBeSet() { $this->validWriter->setImage(array('uri' => 'http://www.example.com/logo.gif')); $atomFeed = new Renderer\Feed\Atom($this->validWriter); $atomFeed->render(); $feed = Reader\Reader::importString($atomFeed->saveXml()); $expected = array('uri' => 'http://www.example.com/logo.gif'); $this->assertEquals($expected, $feed->getImage()); }
/** * Import a feed by providing a URI * * @param string $uri * The URI to the feed * @param string $format * The output format. Possible values xml or json (default). * @return array */ public function feedRead($uri, $format = "json", $fixencoding = true) { $zuri = \Zend\Uri\UriFactory::factory($uri); if (!$zuri->isValid()) { $this->logger->logWarn("Invalid Uri ({$uri})"); return false; } $cache = \Zend\Cache\StorageFactory::factory(array('adapter' => array('name' => 'filesystem', 'options' => array('cache_dir' => __DIR__ . DIRECTORY_SEPARATOR . 'cache', 'ttl' => 3600)), 'plugins' => array(array('name' => 'serializer', 'options' => array())))); \Zend\Feed\Reader\Reader::setCache($cache); \Zend\Feed\Reader\Reader::useHttpConditionalGet(); // Change to firefox agent $httpClient = \Zend\Feed\Reader\Reader::getHttpClient(); $httpClient->setOptions(array('useragent' => 'Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0', 'timeout' => 25)); \Zend\Feed\Reader\Reader::setHttpClient($httpClient); // Import feed try { $feed = \Zend\Feed\Reader\Reader::import($uri); $httpClient = \Zend\Feed\Reader\Reader::getHttpClient(); $httpClientResponse = $httpClient->getResponse(); $feedencoding = $feed->getEncoding(); if (!$feedencoding) { $feedencoding = 'utf-8'; } $feedResponse = array('feed' => $uri, 'statuscode' => $httpClientResponse->getStatusCode(), 'headers' => $httpClientResponse->getHeaders()->toArray(), 'encoding' => $feedencoding); } catch (\Exception $e) { $this->logger->logWarn("Zend feed reader cannot fetch: ({$uri}) because : " . $e->getMessage() . " trying casperjs"); $command = 'casperjs ' . __DIR__ . '/html.js --cookies-file=cookies.txt --url=' . escapeshellarg($uri); exec($command, $response, $return_var); $response = $this->strip_json($response); if ($return_var !== 0 || !isset($response['content']) || empty($response['content'])) { $this->logger->logWarn("Cannot fetch: ({$uri}) exit code {$return_var}", array('command' => $command, 'response' => $response)); $failed = new \Rss\Feed\Reader\ReadFailures($this->postgresql); $failed->update_failure($uri, json_encode(array('zend' => $e->getMessage(), 'casper' => $response))); return false; } try { $feed = \Zend\Feed\Reader\Reader::importString($response['content']); $feedencoding = $feed->getEncoding(); if (!$feedencoding) { $feedencoding = 'utf-8'; } $feedResponse = array('feed' => $uri, 'statuscode' => $response['http']['status'], 'headers' => $response['http'], 'encoding' => $feedencoding); } catch (\Exception $e) { $this->logger->logWarn("Cannot parse feed content from ({$uri}) because " . $e->getMessage()); // $failed = new \Rss\Feed\Reader\ReadFailures ( $this->postgresql ); // $failed->update_failure ( $uri ); return false; } } // Fix relative links $newdata = $this->fix_links($feed, $uri); // Fix encoding errors if ($fixencoding) { $newdata = $this->fix_encoding($newdata); } // Return XML if ($format === "xml") { $feedResponse['data'] = $newdata; return $feedResponse; } // Reload fixed data try { $feed = \Zend\Feed\Reader\Reader::importString($newdata); } catch (\Exception $e) { $this->logger->logWarn("Cannot parse corrected feed content from ({$uri}) because " . $e->getMessage()); return false; } $data = array('title' => $feed->getTitle(), 'link' => $feed->getLink(), 'date' => $this->get_feed_date($feed), 'description' => $feed->getDescription(), 'language' => $feed->getLanguage(), 'entries' => array()); foreach ($feed as $entry) { if (is_object($entry)) { $DateCreated = $entry->getDateCreated(); if (is_object($DateCreated)) { $DateCreated = $DateCreated->getTimestamp(); } else { $DateCreated = $data['date']; } $DateModified = $entry->getDateModified(); if (is_object($DateModified)) { $DateModified = $DateModified->getTimestamp(); } else { $DateModified = $data['date']; } if (empty($DateModified)) { $DateModified = time(); } if (empty($DateCreated)) { $DateCreated = time(); } $description = $entry->getDescription(); $content = $entry->getContent(); if (empty($description)) { $description = $content; } if (empty($content)) { $content = $description; } $edata = array('title' => $entry->getTitle(), 'description' => $description, 'dateCreated' => $DateCreated, 'dateModified' => $DateModified, 'link' => $entry->getLink(), 'content' => $content, 'feed' => $uri); if ($this->check_missing($edata)) { $data['entries'][] = $edata; } else { $this->logger->logTrace("Missing data from feed {$uri} " . json_encode($edata, JSON_UNESCAPED_UNICODE)); } } } // Return array $feedResponse['data'] = $data; return $feedResponse; }
public function testGetsImageFromAtom10_None() { $feed = Reader\Reader::importString(file_get_contents($this->feedSamplePath . '/image/plain/none/atom10.xml')); $this->assertEquals(null, $feed->getImage()); }
/** * Performs a Twitter search query. * * @throws Http\Client\Exception * @return mixed */ public function execute($query, array $params = array()) { $_query = array(); $_query['q'] = $query; foreach ($params as $key => $param) { switch ($key) { case 'geocode': case 'lang': case 'since_id': $_query[$key] = $param; break; case 'rpp': $_query[$key] = intval($param) > 100 ? 100 : intval($param); break; case 'page': $_query[$key] = intval($param); break; case 'show_user': $_query[$key] = 'true'; } } $response = $this->restGet('/search.' . $this->responseType, $_query); switch ($this->responseType) { case 'json': return Json\Json::decode($response->getBody(), Json\Json::TYPE_ARRAY); break; case 'atom': return Feed\Reader\Reader::importString($response->getBody()); break; } return; }