/**
* @inheritdoc
*/
protected function execute(InputInterface $input, OutputInterface $output)
{
$action = $input->getArgument('action');
$number = $input->getArgument('number');
$kicked = $this->manager->kick($action, $number);
$output->writeln(sprintf('Kicked <info>%d</info> buried <info>%s</info> job(s)', $kicked, $action));
}
/**
* @param OutputInterface $output
* @param array $actions
* @param bool $empty
*/
protected function render(OutputInterface $output, array $actions, $empty = false)
{
$fields = ['action' => 'name', 'workers' => 'current-watching', 'reserved' => 'current-jobs-reserved', 'ready' => 'current-jobs-ready', 'urgent' => 'current-jobs-urgent', 'delayed' => 'current-jobs-delayed', 'buried' => 'current-jobs-buried'];
$table = new Table($output);
$table->setHeaders(array_keys($fields));
$rows = [];
foreach ($actions as $action) {
if (!($stats = $this->manager->getActionStats($action))) {
if (!$empty) {
continue;
}
$stats = array_combine(array_values($fields), array_fill(0, sizeof($fields), '-'));
$stats['name'] = $action;
}
$rows[$action] = array_map(function ($field) use($stats) {
return $stats[$field];
}, $fields);
}
ksort($rows);
$table->addRows($rows);
if ($this->lineCount) {
// move back to the beginning
$output->write("[0G");
$output->write(sprintf("[%dA", $this->lineCount));
// overwrite the complete table before rendering the new one
$width = $this->getApplication()->getTerminalDimensions()[0];
$lines = array_fill(0, $this->lineCount, str_pad('', $width, ' '));
$output->writeln($lines);
$output->write(sprintf("[%dA", $this->lineCount));
}
// render the new table
$table->render();
// top table border + header + header border + bottom table border = 4
$this->lineCount = 4 + sizeof($rows);
}
/**
* @inheritdoc
*/
protected function execute(InputInterface $input, OutputInterface $output)
{
$actions = $input->getArgument('action');
if (empty($actions)) {
$actions = array_keys($this->manager->getExecutors());
}
$states = $input->getOption('state');
$pheanstalk = $this->manager->getPheanstalk();
foreach ($actions as $action) {
try {
$stats = $pheanstalk->statsTube($action);
$amount = 0;
foreach ($states as $state) {
$amount += $stats['current-jobs-' . $state];
}
$output->writeln(sprintf('Clearing <info>%d %s jobs</info> with <info>%s</info> status', $amount, $action, implode(', ', $states)));
$this->manager->clear($action, $states);
$output->writeln(['<info>Done!</info>', '']);
} catch (ServerException $e) {
if (false === strpos($e->getMessage(), 'NOT_FOUND')) {
throw $e;
}
}
}
}
/**
* @inheritdoc
*/
protected function execute(InputInterface $input, OutputInterface $output)
{
$action = $input->getArgument('action');
$payload = $input->getArgument('payload');
$output->writeln(sprintf('Performing action <info>%s</info> with payload <info>%s</info>', $action, json_encode($payload, JSON_UNESCAPED_SLASHES)));
$timeStart = microtime(true) * 1000;
$memStart = memory_get_usage(true);
$result = $this->manager->execute($action, $payload);
$duration = microtime(true) * 1000 - $timeStart;
$usage = memory_get_usage(true) - $memStart;
$output->writeln(sprintf('Completed in <comment>%dms</comment> using <comment>%s</comment> with result: <info>%s</info>', $duration, $this->formatBytes($usage), json_encode($result, JSON_UNESCAPED_SLASHES)));
$this->manager->getDispatcher()->dispatch(WorkerEvents::FLUSH);
}
/**
* @param OutputInterface $output
* @param string $action
* @param int $limit
*/
protected function inspect(OutputInterface $output, $action, $limit = null)
{
$output->writeln(sprintf('Inspecting jobs for the <info>%s</info> action', $action));
$this->manager->watchOnly([$action]);
$jobs = [];
while ($job = $this->manager->get(1)) {
$output->writeln(sprintf('<info>%d</info>: <comment>%s</comment>', $job->getId(), $job->getData()));
$jobs[] = $job;
if (null !== $limit && sizeof($jobs) >= $limit) {
break;
}
}
$output->writeln('Releasing the jobs back to the queue, <error>don\'t cancel this action!</error>');
foreach ($jobs as $job) {
$stats = $this->manager->getJobStats($job);
$this->manager->getPheanstalk()->release($job, $stats['pri']);
}
}
/**
* @param OutputInterface $output
* @param array $feeds
* @param integer $minutes
* @param boolean $force
*
* @return integer
*/
protected function scheduleImports(OutputInterface $output, array $feeds, $minutes, $force = false)
{
if (empty($feeds)) {
return 0;
}
$repo = $this->getFeedRepository();
$manager = $this->importManager;
$num = 0;
$factor = $minutes / sizeof($feeds);
foreach ($feeds as $id => $priority) {
/** @var Feed $feed */
$feed = $repo->find($id);
$offset = round($factor * $num++);
$date = new \DateTime(sprintf('+%d minutes', $offset));
$output->writeln(sprintf('Scheduling import for <info>%s</info> feed <info>%d</info> to run at <info>%s</info>', $feed->getOrigin()->getName(), $feed->getId(), $date->format('Y-m-d H:i:s')));
// schedule all resources
try {
$dispatcher = $manager->getEventDispatcher();
$import = $manager->createImport($feed, $date, $force);
$reader = $manager->createImportReader($import, $dispatcher, $manager->getReaderOptions($import));
} catch (\RuntimeException $e) {
// we could get an exception when a new import cannot be created, for example when an existing import
// for this feed is still running.
$output->writeln(sprintf('<error>%s</error>', $e->getMessage()));
continue;
}
$output->writeln(sprintf('Created import <info>%d</info>, adding parts...', $import->getId()));
try {
while ($reader->getResources()->count()) {
$resource = $reader->getResources()->dequeue();
$transportConfig = TransportFactory::createConfigFromTransport($resource->getTransport());
$manager->createImportPart($import, $transportConfig, $date);
$output->writeln(sprintf('=> <comment>%s</comment> (<comment>%d</comment> resources left)', (string) $resource->getTransport(), $reader->getResources()->count()));
}
} catch (FeedException $e) {
$output->writeln(sprintf('<error>%s</error>', $e->getMessage()));
}
if ($import->getParts()->count() === 0) {
$output->writeln('No parts created, finishing import now');
$manager->startImport($import);
$manager->finishImport($import);
continue;
}
$output->write('Scheduling jobs for all parts...');
foreach ($import->getParts() as $part) {
$delay = $date->getTimestamp() - time();
if ($delay < 0) {
$delay = 0;
}
$this->queueManager->addForObject(ImportPartExecutor::NAME, $part, $delay);
}
$output->writeln(' <info>done!</info>');
}
return 0;
}
/**
* @inheritdoc
*/
protected function execute(InputInterface $input, OutputInterface $output)
{
$action = $input->getArgument('action');
$payload = $input->getArgument('payload');
$priority = $input->getOption('priority');
$delay = $input->getOption('delay');
$ttr = $input->getOption('ttr');
$dql = $input->getOption('dql');
if ($payload && $dql) {
throw new \InvalidArgumentException('You cannot provide both a <comment>payload</comment> and a <comment>--dql</comment> query.');
}
if (!empty($payload)) {
$job = $this->manager->add($action, $payload, $delay, $priority, $ttr);
$output->writeln(sprintf('Scheduled job <info>%d</info> with payload <info>%s</info>', $job, json_encode($payload)));
return 0;
}
if (empty($dql)) {
throw new \InvalidArgumentException('You must provide either a <comment>payload</comment> or a <comment>--dql</comment> query.');
}
if (!$this->doctrine) {
$output->writeln('<error>Doctrine is required for the --dql option</error>');
return 1;
}
$doctrine = $this->doctrine->getManager();
if (!$doctrine instanceof EntityManagerInterface) {
$output->writeln('<error>Sorry, only Doctrine\'s ORM is supported at this point. You\'re welcome to submit a PR of course!</error>');
return 1;
}
$meta = null;
$query = $doctrine->createQuery($dql);
foreach ($query->iterate() as list($entity)) {
if (!$meta) {
$meta = $doctrine->getClassMetadata(get_class($entity));
}
$job = $this->manager->addForObject($action, $entity, $delay, $priority, $ttr);
$output->writeln(sprintf('Scheduled job <info>%d</info> for entity <info>%s: %s</info>', $job, json_encode($meta->getIdentifierValues($entity)), $this->entityToString($entity)));
$doctrine->clear();
}
return 0;
}
public function testExecuteJobAndNoMoreRetries()
{
$action = 'test';
$executor = $this->getMockForAbstractClass(ExecutorInterface::class);
$executor->expects($this->any())->method('getName')->will($this->returnValue($action));
$this->manager->addExecutor($executor);
$data = [];
$job = new Job(1234, json_encode($data));
$stats = ['tube' => $action, 'releases' => 2, 'pri' => PheanstalkInterface::DEFAULT_PRIORITY];
$this->pheanstalk->expects($this->once())->method('statsJob')->with($job)->will($this->returnValue($stats));
$this->pheanstalk->expects($this->once())->method('bury')->with($job);
$executor->expects($this->once())->method('execute')->will($this->throwException(new \Exception('oh noes!')));
$this->manager->executeJob($job);
}
/**
* @inheritdoc
*/
protected function execute(InputInterface $input, OutputInterface $output)
{
$actions = $input->getArgument('action');
if (empty($actions)) {
$actions = array_keys($this->manager->getExecutors());
}
$states = $input->getOption('state');
foreach ($actions as $action) {
$output->writeln(sprintf('Action: <info>%s</info>', $action));
$table = new Table($output);
$table->setHeaders(['state', 'id', 'data']);
foreach ($states as $state) {
$job = $this->manager->peek($action, $state);
if (is_null($job)) {
$table->addRow([$state, '-', '-']);
} else {
$table->addRow([$state, $job->getId(), $job->getData()]);
}
}
$table->render();
$output->writeln('');
}
}
/**
* @param string[] $include
* @param string[] $exclude
*/
protected function watchActions(array $include = [], array $exclude = [])
{
$actions = array_keys($this->manager->getExecutors());
if (empty($include)) {
$include = $actions;
}
if (!empty($diff = array_diff($include, $actions))) {
throw new \InvalidArgumentException(sprintf('Action(s) "%s" are not defined by QueueManager', implode(', ', $diff)));
}
if (!empty($diff = array_diff($exclude, $actions))) {
throw new \InvalidArgumentException(sprintf('Filter(s) "%s" are not defined by QueueManager', implode(', ', $diff)));
}
$include = array_diff($include, $exclude);
if (empty($include)) {
throw new \InvalidArgumentException('No actions specified to run');
}
// watch only these actions
$this->manager->watchOnly($include);
}
/**
* @param SourceEvent $event
*/
public function onSourceProcess(SourceEvent $event)
{
$this->queueManager->addForObject(SourceProcessExecutor::NAME, $event->getSource());
}
/**
* @param RateLimitEvent $event
*/
public function onRateLimit(RateLimitEvent $event)
{
$payload = [$event->getScraper()->getId(), $event->getUrl()];
$this->queueManager->add(ScrapeUrlExecutor::NAME, $payload, $event->getRetryDate()->getTimestamp() - time());
}
/**
* @param PartEvent $event
*/
public function onPartScheduled(PartEvent $event)
{
$this->queueManager->addForObject(ImportPartExecutor::NAME, $event->getPart(), null, null, $this->timeToRun);
}
/**
* Schedules a source process job
*
* @param SourceInterface $source
*/
protected function scheduleSourceProcess(SourceInterface $source)
{
$this->queueManager->addForObject(SourceProcessExecutor::NAME, $source);
}
/**
* @param ScrapeUrlEvent $event
*/
public function onScrapeNextUrl(ScrapeUrlEvent $event)
{
$this->queueManager->add(ScrapeUrlExecutor::NAME, [$event->getScraper()->getId(), $event->getUrl()]);
}
/**
* @param SourceEvent $event
*/
public function onScrapeRevisitSource(SourceEvent $event)
{
$this->queueManager->addForObject(ScrapeRevisitSourceExecutor::NAME, $event->getSource());
}