protected function getFeed($url, $node)
{
$transport = TransportFactory::createTransportFromUrl($url);
$reader = new XmlReader(new FileResource($transport));
$reader->setNodeCallback($node);
return new Feed($reader);
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$id = $input->getArgument('id');
$force = $input->getOption('force');
$manager = $this->importManager;
$doctrine = $manager->getEntityManager();
/** @var $feed Feed */
if (null === ($feed = $doctrine->find('FMIoBundle:Feed', $id))) {
$output->writeln(sprintf('<error>Feed with id "%d" does not exist</error>', $id));
return 1;
}
if ($input->isInteractive()) {
$this->checkForUnfinishedImports($feed, $output);
}
$manager->getLogger()->getLogger()->pushHandler(new ConsoleHandler($output));
$this->setOutputEvents($manager->getEventDispatcher(), $output);
// schedule all resources
$dispatcher = $manager->getEventDispatcher();
$import = $manager->createImport($feed, null, $force);
$reader = $manager->createImportReader($import, $dispatcher, $manager->getReaderOptions($import));
$output->writeln(sprintf('Created import with id <info>%d</info>, start importing parts', $import->getId()));
try {
while ($reader->getResources()->count()) {
$resource = $reader->getResources()->dequeue();
$transportConfig = TransportFactory::createConfigFromTransport($resource->getTransport());
$manager->createImportPart($import, $transportConfig);
$output->writeln(sprintf('=> <comment>%s</comment> (<comment>%d</comment> resources left)', (string) $resource->getTransport(), $reader->getResources()->count()));
}
} catch (FeedException $e) {
$output->writeln(sprintf('<error>%s</error>', $e->getMessage()));
}
foreach ($import->getParts() as $part) {
$output->writeln(sprintf('Importing part <comment>%d</comment>', $part->getPosition()));
$manager->import($part);
}
return 0;
}
/**
* @param array $transportConfig
* @param string $destinationDir
* @param EventDispatcherInterface $dispatcher
* @return Transport
*/
protected function createTransport(array $transportConfig, $destinationDir, EventDispatcherInterface $dispatcher)
{
$transport = TransportFactory::createTransportFromConfig($transportConfig);
$transport->setDestinationDir($destinationDir);
$transport->setEventDispatcher($dispatcher);
return $transport;
}
/**
* @param OutputInterface $output
* @param array $feeds
* @param integer $minutes
* @param boolean $force
*
* @return integer
*/
protected function scheduleImports(OutputInterface $output, array $feeds, $minutes, $force = false)
{
if (empty($feeds)) {
return 0;
}
$repo = $this->getFeedRepository();
$manager = $this->importManager;
$num = 0;
$factor = $minutes / sizeof($feeds);
foreach ($feeds as $id => $priority) {
/** @var Feed $feed */
$feed = $repo->find($id);
$offset = round($factor * $num++);
$date = new \DateTime(sprintf('+%d minutes', $offset));
$output->writeln(sprintf('Scheduling import for <info>%s</info> feed <info>%d</info> to run at <info>%s</info>', $feed->getOrigin()->getName(), $feed->getId(), $date->format('Y-m-d H:i:s')));
// schedule all resources
try {
$dispatcher = $manager->getEventDispatcher();
$import = $manager->createImport($feed, $date, $force);
$reader = $manager->createImportReader($import, $dispatcher, $manager->getReaderOptions($import));
} catch (\RuntimeException $e) {
// we could get an exception when a new import cannot be created, for example when an existing import
// for this feed is still running.
$output->writeln(sprintf('<error>%s</error>', $e->getMessage()));
continue;
}
$output->writeln(sprintf('Created import <info>%d</info>, adding parts...', $import->getId()));
try {
while ($reader->getResources()->count()) {
$resource = $reader->getResources()->dequeue();
$transportConfig = TransportFactory::createConfigFromTransport($resource->getTransport());
$manager->createImportPart($import, $transportConfig, $date);
$output->writeln(sprintf('=> <comment>%s</comment> (<comment>%d</comment> resources left)', (string) $resource->getTransport(), $reader->getResources()->count()));
}
} catch (FeedException $e) {
$output->writeln(sprintf('<error>%s</error>', $e->getMessage()));
}
if ($import->getParts()->count() === 0) {
$output->writeln('No parts created, finishing import now');
$manager->startImport($import);
$manager->finishImport($import);
continue;
}
$output->write('Scheduling jobs for all parts...');
foreach ($import->getParts() as $part) {
$delay = $date->getTimestamp() - time();
if ($delay < 0) {
$delay = 0;
}
$this->queueManager->addForObject(ImportPartExecutor::NAME, $part, $delay);
}
$output->writeln(' <info>done!</info>');
}
return 0;
}
/**
* @param Feed $feed
* @param string $fixtureName
* @param EventDispatcherInterface $dispatcher
* @param array $options
*
* @return ReaderInterface
*/
protected function createReader(Feed $feed, $fixtureName, EventDispatcherInterface $dispatcher, array $options = [])
{
$readerType = $this->getImportRegistry()->getReaderType($feed->getReaderType());
$refl = new \ReflectionClass(get_class($this));
$xml = sprintf('%s/fixtures/%s/%s.xml', dirname($refl->getFileName()), $feed->getType(), $fixtureName);
$transportConfig = TransportFactory::createConfigFromFile($xml);
$builder = new ReaderBuilder($dispatcher, sys_get_temp_dir() . '/' . $feed->getType());
return $builder->build($readerType, $transportConfig, $builder::RESOURCE_TYPE_MAIN, $options);
}
