Example #1
0
 /**
  * {@inheritdoc}
  */
 public function fetch(FeedInterface $feed, StateInterface $state)
 {
     $path = $feed->getSource();
     // Just return a file fetcher result if this is a file. Make sure to
     // re-validate the file extension in case the feed type settings have
     // changed.
     if (is_file($path)) {
         if ($this->validateFilePath($path)) {
             return new FetcherResult($path);
         } else {
             throw new \RuntimeException($this->t('%source has an invalid file extension.', ['%source' => $path]));
         }
     }
     if (!is_dir($path) || !is_readable($path)) {
         throw new \RuntimeException($this->t('%source is not a readable directory or file.', ['%source' => $path]));
     }
     // Batch if this is a directory.
     if (!isset($state->files)) {
         $state->files = $this->listFiles($path);
         $state->total = count($state->files);
     }
     if ($state->files) {
         $file = array_shift($state->files);
         $state->progress($state->total, $state->total - count($state->files));
         return new FetcherResult($file);
     }
     throw new EmptyFeedException();
 }
Example #2
0
 /**
  * {@inheritdoc}
  */
 public function parse(FeedInterface $feed, FetcherResultInterface $fetcher_result, StateInterface $state)
 {
     $feed_config = $feed->getConfigurationFor($this);
     if (!filesize($fetcher_result->getFilePath())) {
         throw new EmptyFeedException();
     }
     // Load and configure parser.
     $parser = CsvFileParser::createFromFilePath($fetcher_result->getFilePath())->setDelimiter($feed_config['delimiter'] === 'TAB' ? "\t" : $feed_config['delimiter'])->setHasHeader(!$feed_config['no_headers'])->setStartByte((int) $state->pointer);
     // Wrap parser in a limit iterator.
     $parser = new \LimitIterator($parser, 0, $this->configuration['line_limit']);
     $header = !$feed_config['no_headers'] ? $parser->getHeader() : [];
     $result = new ParserResult();
     foreach ($parser as $row) {
         $item = new DynamicItem();
         foreach ($row as $delta => $cell) {
             $key = isset($header[$delta]) ? $header[$delta] : $delta;
             $item->set($key, $cell);
         }
         $result->addItem($item);
     }
     // Report progress.
     $state->total = filesize($fetcher_result->getFilePath());
     $state->pointer = $parser->lastLinePos();
     $state->progress($state->total, $state->pointer);
     return $result;
 }
Example #3
0
 /**
  * {@inheritdoc}
  */
 public function postClear(FeedInterface $feed, StateInterface $state)
 {
     $tokens = ['@item' => $this->getItemLabel(), '@items' => $this->getItemLabelPlural(), '%title' => $feed->label()];
     if ($state->deleted) {
         $state->setMessage($this->formatPlural($state->deleted, 'Deleted @count @item from %title.', 'Deleted @count @items from %title.', $tokens));
     } else {
         $state->setMessage($this->t('There are no @items to delete.', $tokens));
     }
 }
Example #4
0
 /**
  * {@inheritdoc}
  */
 public function fetch(FeedInterface $feed, StateInterface $state)
 {
     $sink = $this->generateSink();
     $configuration = $this->getConfiguration();
     $yt_state = ['channel_id' => $feed->getSource(), 'api_key' => $configuration['api_key'], 'import_limit' => $configuration['import_limit'], 'page_limit' => $configuration['page_limit'], 'pageToken' => ''];
     $url = FeedsYoutubeHandler::buildURL($yt_state);
     $response = $this->get($url, $sink, $this->getCacheKey($feed));
     // 304, nothing to see here.
     if ($response->getStatusCode() == Response::HTTP_NOT_MODIFIED) {
         $state->setMessage($this->t('The feed has not been updated.'));
         throw new EmptyFeedException();
     }
     return new HttpFetcherResult($sink, $response->getHeaders());
 }
Example #5
0
 /**
  * {@inheritdoc}
  */
 public function fetch(FeedInterface $feed, StateInterface $state)
 {
     $response = $this->get($feed->getSource(), $this->getCacheKey($feed));
     $feed->setSource($response->getEffectiveUrl());
     // 304, nothing to see here.
     if ($response->getStatusCode() == 304) {
         $state->setMessage($this->t('The feed has not been updated.'));
         throw new EmptyFeedException();
     }
     // Copy the temp stream to a real file.
     $download_file = drupal_tempnam('temporary://', 'feeds_http_fetcher');
     $dest_stream = Utils::create(fopen($download_file, 'w+'));
     Utils::copyToStream($response->getBody(), $dest_stream);
     $response->getBody()->close();
     $dest_stream->close();
     return new HttpFetcherResult($download_file, $response->getHeaders());
 }
Example #6
0
 /**
  * {@inheritdoc}
  */
 public function clear(FeedInterface $feed, StateInterface $state)
 {
     // Build base select statement.
     $query = $this->queryFactory->get($this->entityType())->condition('feeds_item.target_id', $feed->id());
     // If there is no total, query it.
     if (!$state->total) {
         $count_query = clone $query;
         $state->total = (int) $count_query->count()->execute();
     }
     // Delete a batch of entities.
     $entity_ids = $query->range(0, 10)->execute();
     if ($entity_ids) {
         $this->entityDeleteMultiple($entity_ids);
         $state->deleted += count($entity_ids);
         $state->progress($state->total, $state->deleted);
     }
 }