/** * {@inheritdoc} */ public function fetch(FeedInterface $feed) { $request = $this->httpClient->createRequest('GET', $feed->getUrl()); $feed->source_string = FALSE; // Generate conditional GET headers. if ($feed->getEtag()) { $request->addHeader('If-None-Match', $feed->getEtag()); } if ($feed->getLastModified()) { $request->addHeader('If-Modified-Since', gmdate(DateTimePlus::RFC7231, $feed->getLastModified())); } try { $response = $this->httpClient->send($request); // In case of a 304 Not Modified, there is no new content, so return // FALSE. if ($response->getStatusCode() == 304) { return FALSE; } $feed->source_string = $response->getBody(TRUE); $feed->setEtag($response->getHeader('ETag')); $feed->setLastModified(strtotime($response->getHeader('Last-Modified'))); $feed->http_headers = $response->getHeaders(); // Update the feed URL in case of a 301 redirect. if ($response->getEffectiveUrl() != $feed->getUrl()) { $feed->setUrl($response->getEffectiveUrl()); } return TRUE; } catch (RequestException $e) { watchdog('aggregator', 'The feed from %site seems to be broken because of error "%error".', array('%site' => $feed->label(), '%error' => $e->getMessage()), WATCHDOG_WARNING); drupal_set_message(t('The feed from %site seems to be broken because of error "%error".', array('%site' => $feed->label(), '%error' => $e->getMessage())), 'warning'); return FALSE; } }
/** * {@inheritdoc} */ public function refresh(FeedInterface $feed) { // Store feed URL to track changes. $feed_url = $feed->getUrl(); // Fetch the feed. try { $success = $this->fetcherManager->createInstance($this->config->get('fetcher'))->fetch($feed); } catch (PluginException $e) { $success = FALSE; watchdog_exception('aggregator', $e); } // Store instances in an array so we dont have to instantiate new objects. $processor_instances = array(); foreach ($this->config->get('processors') as $processor) { try { $processor_instances[$processor] = $this->processorManager->createInstance($processor); } catch (PluginException $e) { watchdog_exception('aggregator', $e); } } // We store the hash of feed data in the database. When refreshing a // feed we compare stored hash and new hash calculated from downloaded // data. If both are equal we say that feed is not updated. $hash = hash('sha256', $feed->source_string); $has_new_content = $success && $feed->getHash() != $hash; if ($has_new_content) { // Parse the feed. try { if ($this->parserManager->createInstance($this->config->get('parser'))->parse($feed)) { if (!$feed->getWebsiteUrl()) { $feed->setWebsiteUrl($feed->getUrl()); } $feed->setHash($hash); // Update feed with parsed data. $feed->save(); // Log if feed URL has changed. if ($feed->getUrl() != $feed_url) { $this->logger->notice('Updated URL for feed %title to %url.', array('%title' => $feed->label(), '%url' => $feed->getUrl())); } $this->logger->notice('There is new syndicated content from %site.', array('%site' => $feed->label())); // If there are items on the feed, let enabled processors process them. if (!empty($feed->items)) { foreach ($processor_instances as $instance) { $instance->process($feed); } } } } catch (PluginException $e) { watchdog_exception('aggregator', $e); } } // Processing is done, call postProcess on enabled processors. foreach ($processor_instances as $instance) { $instance->postProcess($feed); } return $has_new_content; }
/** * {@inheritdoc} */ public function getFeedDuplicates(FeedInterface $feed) { $query = \Drupal::entityQuery('aggregator_feed'); $or_condition = $query->orConditionGroup()->condition('title', $feed->label())->condition('url', $feed->getUrl()); $query->condition($or_condition); if ($feed->id()) { $query->condition('fid', $feed->id(), '<>'); } return $this->loadMultiple($query->execute()); }
/** * {@inheritdoc} */ public function fetch(FeedInterface $feed) { $request = new Request('GET', $feed->getUrl()); $feed->source_string = FALSE; // Generate conditional GET headers. if ($feed->getEtag()) { $request = $request->withAddedHeader('If-None-Match', $feed->getEtag()); } if ($feed->getLastModified()) { $request = $request->withAddedHeader('If-Modified-Since', gmdate(DateTimePlus::RFC7231, $feed->getLastModified())); } try { /** @var \Psr\Http\Message\UriInterface $actual_uri */ $actual_uri = NULL; $response = $this->httpClientFactory->fromOptions(['allow_redirects' => ['on_redirect' => function (RequestInterface $request, ResponseInterface $response, UriInterface $uri) use(&$actual_uri) { $actual_uri = (string) $uri; }]])->send($request); // In case of a 304 Not Modified, there is no new content, so return // FALSE. if ($response->getStatusCode() == 304) { return FALSE; } $feed->source_string = (string) $response->getBody(); if ($response->hasHeader('ETag')) { $feed->setEtag($response->getHeaderLine('ETag')); } if ($response->hasHeader('Last-Modified')) { $feed->setLastModified(strtotime($response->getHeaderLine('Last-Modified'))); } $feed->http_headers = $response->getHeaders(); // Update the feed URL in case of a 301 redirect. if ($actual_uri && $actual_uri !== $feed->getUrl()) { $feed->setUrl($actual_uri); } return TRUE; } catch (RequestException $e) { $this->logger->warning('The feed from %site seems to be broken because of error "%error".', array('%site' => $feed->label(), '%error' => $e->getMessage())); drupal_set_message(t('The feed from %site seems to be broken because of error "%error".', array('%site' => $feed->label(), '%error' => $e->getMessage())), 'warning'); return FALSE; } }
/** * Updates the feed items. * * This method simulates a click to * admin/config/services/aggregator/update/$fid. * * @param \Drupal\aggregator\FeedInterface $feed * Feed object representing the feed. * @param int|null $expected_count * Expected number of feed items. If omitted no check will happen. */ public function updateFeedItems(FeedInterface $feed, $expected_count = NULL) { // First, let's ensure we can get to the rss xml. $this->drupalGet($feed->getUrl()); $this->assertResponse(200, format_string(':url is reachable.', array(':url' => $feed->getUrl()))); // Attempt to access the update link directly without an access token. $this->drupalGet('admin/config/services/aggregator/update/' . $feed->id()); $this->assertResponse(403); // Refresh the feed (simulated link click). $this->drupalGet('admin/config/services/aggregator'); $this->clickLink('Update items'); // Ensure we have the right number of items. $result = db_query('SELECT iid FROM {aggregator_item} WHERE fid = :fid', array(':fid' => $feed->id())); $feed->items = array(); foreach ($result as $item) { $feed->items[] = $item->iid; } if ($expected_count !== NULL) { $feed->item_count = count($feed->items); $this->assertEqual($expected_count, $feed->item_count, format_string('Total items in feed equal to the total items in database (@val1 != @val2)', array('@val1' => $expected_count, '@val2' => $feed->item_count))); } }