/**
* Run the command.
*
* @param InputInterface $input
* @param OutputInterface $output
*
* @return void
* @throws Exception
*/
protected function execute(InputInterface $input, OutputInterface $output)
{
$serviceName = $input->getArgument('service');
$service = $this->serviceManager->getService($serviceName);
$hostProvider = $service->getHostProvider();
$this->getApplication()->find('listhosts')->run(new ArrayInput(['service' => $serviceName]), $output);
$output->writeln("");
$progress = new ProgressBar($output, 5);
$progress->setFormat(' %current%/%max% [%bar%] %percent:3s%% %message%');
$progress->setMessage('Checking existing hosts');
$progress->start();
$currentHostCount = count($hostProvider->getHostsForService($service));
if ($service->maxHosts && $currentHostCount >= $service->maxHosts) {
throw new Exception("There are already {$currentHostCount}/{$service->maxHosts} hosts for {$serviceName}.");
}
$newInstance = $currentHostCount + 1;
$hostname = sprintf($service->hostnameTemplate, $newInstance);
$host = $hostProvider->launch($hostname, $service->hostDefaults);
$hostname = $host->getHostname();
// Just check it set the right name
$progress->setMessage("Created host " . $hostname . " at " . $hostProvider->getName());
$progress->advance();
sleep(5);
$progress->setMessage("Waiting for " . $hostname . " to be ready");
$progress->advance();
while (!$host->isReady()) {
$lastState = $host->getState();
$progress->setMessage("Waiting for " . $hostname . " to be ready (Current sate: " . $lastState . ")");
$progress->display();
sleep(10);
}
if (!empty($service->testUrl)) {
$progress->setMessage("Testing host's HTTP response");
$progress->advance();
do {
$lastResponse = $host->testHttp($service->testUrl, $service->testUrlHeaders);
$progress->setMessage("Testing host's HTTP response (Current response: {$lastResponse})");
$progress->display();
$lastResponse === 200 || sleep(5);
} while ($lastResponse !== 200);
}
$dnsProvider = $service->getDnsProvider();
$recordData = [];
foreach ($host->publicIps as $ip) {
foreach ($service->dnsRecords as $domain => $domainRecords) {
foreach ($domainRecords as $record => $recordSettings) {
$data = ['domain' => $domain, 'type' => $ip->version === 6 ? 'AAAA' : 'A', 'name' => sprintf($record, $newInstance), 'value' => $ip->ip];
$recordData[] = $data + $recordSettings;
}
}
}
$progress->setMessage("Adding " . count($recordData) . " DNS records to " . $dnsProvider->getName());
$progress->advance();
$dnsProvider->addRecords($recordData);
$progress->setMessage('Done!');
$progress->finish();
$output->writeln("");
$output->writeln("");
$this->getApplication()->find('listhosts')->run(new ArrayInput(['service' => $serviceName]), $output);
}
public function showProgressIndicator()
{
if ($this->progressIndicatorRunning && !$this->progressBarDisplayed && isset($this->progressBar)) {
$this->progressBar->display();
$this->progressBarDisplayed = true;
$this->advanceProgressIndicatorCachedSteps();
}
}
/**
* @param int $current
* @param PhpFileInfo $file
*/
public function advance($current, PhpFileInfo $file)
{
if (!$this->verbose) {
return;
}
if (1 === $current) {
$format = '<info>%message%</info>' . "\n" . $this->label . ': <info>%current%</info>/<info>%max%</info>';
$this->progressBar->clear();
$this->progressBar->setFormat($format);
}
$message = $file->getRelativePathname();
$this->progressBar->setMessage($message);
$this->progressBar->clear();
$this->progressBar->advance();
$this->progressBar->display();
}
/**
* {@inheritdoc}
*/
protected function execute(InputInterface $input, OutputInterface $output)
{
if (!Loader::includeModule('search')) {
throw new BitrixException('Search module is not installed');
}
$searchResult = array();
$bar = new ProgressBar($output, 0);
do {
$bar->display();
$searchResult = \CSearch::ReIndexAll($input->getOption('full'), static::UPDATE_TIME, $searchResult);
$bar->advance();
$bar->clear();
if (is_array($searchResult) && $searchResult['MODULE'] == 'main') {
list(, $path) = explode("|", $searchResult["ID"], 2);
$output->writeln("\r " . $path, OutputInterface::VERBOSITY_VERBOSE);
}
} while (is_array($searchResult));
$bar->finish();
$bar->clear();
$output->write("\r");
if (ModuleManager::isModuleInstalled('socialnetwork')) {
$output->writeln('<info>The Social Network module needs to be reindexed using the Social Network component in the public section of site.</info>');
}
$output->writeln(sprintf('<info>Reindexed</info> %d element%s.', $searchResult, $searchResult > 1 ? 's' : ''));
return 0;
}
/**
* TODO change interface to method passing in configuration object (which is validated)
*
* Perform a rate-limited API call. The flow is:
* 1. requestFunction()
* 2. processFunction() based on requestFunction result
* 3. publishFunction() based on processFunction result
*
* Only requestFunction() and serviceName are required fields.
*
* @param $serviceName
* @param callable $requestFunction should return a list for processing
* @param callable $processFunction must return a list of models for publishing
* @param callable $publishFunction method to upload models; responsible for handling publication failures
* @return mixed
*/
public function makeRateLimitedRequest($serviceName, $requestFunction, $processFunction = null, $publishFunction = null)
{
$rateLimit = self::$rate_limits[$serviceName];
if (SyncCommandBase::$requests_processed_this_minute[$serviceName] >= $rateLimit) {
$seconds_to_sleep = 60 - (time() - SyncCommandBase::$start_of_minute_timestamp[$serviceName]);
if ($seconds_to_sleep > 0) {
$this->progressBar->setMessage("Rate limit reached for '{$serviceName}'. Waiting {$seconds_to_sleep} seconds.");
$this->progressBar->display();
sleep($seconds_to_sleep);
$this->progressBar->setMessage("");
}
SyncCommandBase::$start_of_minute_timestamp[$serviceName] = time();
SyncCommandBase::$requests_processed_this_minute[$serviceName] = 0;
} elseif (time() - SyncCommandBase::$start_of_minute_timestamp[$serviceName] > 60) {
SyncCommandBase::$start_of_minute_timestamp[$serviceName] = time();
SyncCommandBase::$requests_processed_this_minute[$serviceName] = 0;
}
$response = $requestFunction();
SyncCommandBase::$requests_processed_this_minute[$serviceName]++;
if ($processFunction != null) {
/** @var callable $processFunction */
$processedModels = $processFunction($response);
if ($publishFunction != null) {
/** @var callable $publishFunction */
$publishFunction($processedModels);
}
} else {
// don't do anything
}
return $response;
}
/**
* Set progress bar. Once a progress bar has been set on an invoker, it
* cannot be undone. Instantiate a new invoker if needed. Sub sequent calls
* to `Invoker::execute()` will reuse the progress bar object.
*
* @param ProgressBar $bar
* @return Invoker
*/
public function setProgressBar(ProgressBar $bar)
{
// start progress bar
$this->dispatcher->addListener(Events::INVOKER_START, function (InvokerEvent $event) use($bar) {
$bar->start($event->getSteps()->getUnits());
});
// finish progress bar
$this->dispatcher->addListener(Events::INVOKER_FINISH, function (InvokerEvent $event) use($bar) {
$bar->setMessage('Finished: ' . $event->getSteps()->getName());
$bar->finish();
});
// step start
$this->dispatcher->addListener(Events::STEP_BEFORE_EXECUTE, function (StepEvent $event) use($bar) {
$bar->setMessage($event->getStep()->getDescription() . '...');
$bar->display();
});
// step finish
$this->dispatcher->addListener(Events::STEP_AFTER_EXECUTE, function (StepEvent $event) use($bar) {
$bar->advance($event->getStep()->getTicksRemaining());
});
// step tick
$this->dispatcher->addListener(Events::STEP_TICK, function (StepEvent $event) use($bar) {
$bar->advance($event->getStep()->getTick());
});
return $this;
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$io = new SymfonyStyle($input, $output);
$io->title('Exporting databases');
$io->section('Exporting all databases');
$strategies = $this->collectorDbStrategy->collectDatabasesStrategies();
$totalStrategies = count($strategies);
$io->writeln($totalStrategies . ' strategie(s) found.');
$progressBar = new ProgressBar($output, $totalStrategies);
$progressBar->setFormat(self::PROGRESS_BAR_FORMAT);
$progressBar->setMessage('Beginning backuping');
$this->eventDispatcher->dispatch(Events::BACKUP_BEGINS, new BackupBeginsEvent($output));
$progressBar->start();
$reportContent = new \ArrayObject();
foreach ($strategies as $strategy) {
$strategyIdentifier = $strategy->getIdentifier();
$setProgressBarMessage = function ($message) use($progressBar, $strategyIdentifier) {
$message = "[{$strategyIdentifier}] {$message}";
$progressBar->setMessage($message);
$progressBar->display();
};
$exportedFiles = $this->processorDatabaseDumper->dump($strategy, $setProgressBarMessage);
$reportContent->append("Backuping of the database: {$strategyIdentifier}");
foreach ($exportedFiles as $file) {
$filename = $file->getPath();
$reportContent->append("\t→ {$filename}");
}
$progressBar->advance();
}
$progressBar->finish();
$io->newLine(2);
$io->section('Report');
$io->text($reportContent->getArrayCopy());
$this->eventDispatcher->dispatch(Events::BACKUP_ENDS, new BackupEndsEvent($output));
}
/**
* {@inheritdoc
*/
public function add_success_message($error_title, $error_description = false)
{
$this->io->newLine();
$message = $this->translate_message($error_title, $error_description);
$this->io->success($message['title'] . "\n" . $message['description']);
if ($this->progress_bar !== null) {
$this->io->newLine(2);
$this->progress_bar->display();
}
}
public function run()
{
foreach ($this->processes as $process) {
/** @var $process Process **/
$process->setIdleTimeout($this->idleTimeout);
$process->setTimeout($this->timeout);
$process->start();
$this->printTaskInfo($process->getCommandLine());
}
$progress = new ProgressBar($this->getOutput());
$progress->start(count($this->processes));
$running = $this->processes;
$progress->display();
$this->startTimer();
while (true) {
foreach ($running as $k => $process) {
try {
$process->checkTimeout();
} catch (ProcessTimedOutException $e) {
}
if (!$process->isRunning()) {
$progress->advance();
if ($this->isPrinted) {
$this->getOutput()->writeln("");
$this->printTaskInfo("Output for <fg=white;bg=magenta> " . $process->getCommandLine() . " </fg=white;bg=magenta>");
$this->getOutput()->writeln($process->getOutput(), OutputInterface::OUTPUT_RAW);
if ($process->getErrorOutput()) {
$this->getOutput()->writeln("<error>" . $process->getErrorOutput() . "</error>");
}
}
unset($running[$k]);
}
}
if (empty($running)) {
break;
}
usleep(1000);
}
$this->getOutput()->writeln("");
$this->stopTimer();
$errorMessage = '';
$exitCode = 0;
foreach ($this->processes as $p) {
if ($p->getExitCode() === 0) {
continue;
}
$errorMessage .= "'" . $p->getCommandLine() . "' exited with code " . $p->getExitCode() . " \n";
$exitCode = max($exitCode, $p->getExitCode());
}
if (!$errorMessage) {
$this->printTaskSuccess(count($this->processes) . " processes finished running");
}
return new Result($this, $exitCode, $errorMessage, ['time' => $this->getExecutionTime()]);
}
/**
* Builds a loggerClosure to be called from inside the Provider to update the command
* line.
*
* @param OutputInterface $output
* @param string $action
* @param string $index
* @param string $type
*
* @return callable
*/
public function build(OutputInterface $output, $action, $index, $type)
{
if (!class_exists('Symfony\\Component\\Console\\Helper\\ProgressBar') || !is_callable(array('Symfony\\Component\\Console\\Helper\\ProgressBar', 'getProgress'))) {
return $this->buildLegacy($output, $action, $index, $type);
}
$progress = null;
return function ($increment, $totalObjects, $message = null) use(&$progress, $output, $action, $index, $type) {
if (null === $progress) {
$progress = new ProgressBar($output, $totalObjects);
$progress->start();
}
if (null !== $message) {
$progress->clear();
$output->writeln(sprintf('<info>%s</info> <error>%s</error>', $action, $message));
$progress->display();
}
$progress->setMessage(sprintf('<info>%s</info> <comment>%s/%s</comment>', $action, $index, $type));
$progress->advance($increment);
};
}
public function execute(InputInterface $input, OutputInterface $output)
{
$db = $input->getOption('db') === true;
$db_path = false;
if ($db) {
$db_path = $this->config->db . $this->getTimestamp() . '.sql';
$command = new Database();
$arguments = ['--export' => $db_path];
$arguments = new ArrayInput($arguments);
$command->run($arguments, $output);
$db_path = realpath($db_path);
}
$output->writeln('<info>Scanning files</info>');
$files = $this->getJoomlaFiles($db_path, [getcwd() . '/.private']);
$output->writeln("<info>Archiving files</info>");
$path = getcwd() . DIRECTORY_SEPARATOR . $this->name . $this->getTimestamp() . '.zip';
$zip = new \ZipArchive();
if (true !== $zip->open($path, \ZipArchive::CREATE)) {
throw new \RuntimeException("Can't open {$path}.");
}
$progress = new ProgressBar($output, count($files));
$progress->display();
foreach ($files as $file) {
if (!$file->isDir()) {
$realpath = $file->getRealPath();
$localname = str_replace(getcwd() . DIRECTORY_SEPARATOR, '', $realpath);
$zip->addFile($realpath, $localname);
}
$progress->advance();
}
$progress->finish();
$output->writeln('');
$output->write("<info>Finalizing archive...</info>");
$zip->close();
if ($db_path) {
unlink($db_path);
}
$output->writeln(" <info>Done: " . basename($path) . "</info>");
}
/**
* {@inheritdoc}
*/
protected function execute(InputInterface $input, OutputInterface $output)
{
$batchSize = $input->getOption('batch-size');
/** @var EntityManagerInterface $entityManager */
$entityManager = $this->getContainer()->get('doctrine.orm.entity_manager');
$routeManager = $this->getContainer()->get('sulu_route.manager.route_manager');
/** @var EntityRepository $repository */
$repository = $entityManager->getRepository($input->getArgument('entity'));
$query = $repository->createQueryBuilder('entity')->select('count(entity.id)')->getQuery();
$result = $query->getResult();
$count = (int) $result[0][1];
$query = $repository->createQueryBuilder('entity')->getQuery();
$output->writeln(sprintf('<comment>updating route for "%s" instances of "%s"</comment>', $count, $input->getArgument('entity')));
$progressBar = new ProgressBar($output, $count);
$progressBar->setFormat('debug');
$progressBar->display();
$index = 0;
foreach ($query->iterate() as $item) {
$entity = $item[0];
if (null !== $entity->getRoute()) {
$routeManager->update($entity);
} else {
$entityManager->persist($routeManager->create($entity));
}
$progressBar->advance();
$entity = null;
if (0 === $index++ % $batchSize) {
$entityManager->flush();
// trigger garbage collect
$entityManager->clear();
}
}
// flush the rest of the entities
$entityManager->flush();
//$progressBar->finish();
$output->writeln('');
}
/**
* @param int $width
*
* @return ProgressBar
*/
private function createProgressBar($width)
{
$progress = new ProgressBar($this->output, $width);
$progress->setBarWidth(50);
$progress->display();
return $progress;
}
$phpExcel->getActiveSheet()->setCellValue('B' . $i, $book['authors']);
$phpExcel->getActiveSheet()->setCellValue('C' . $i, $book['publisher']);
$phpExcel->getActiveSheet()->setCellValue('D' . $i, $book['description']);
$phpExcel->getActiveSheet()->setCellValue('E' . $i, $book['pageCount']);
$phpExcel->getActiveSheet()->setCellValue('F' . $i, $book['imageLink']);
$i++;
} catch (BookNotFoundException $e) {
$app['monolog']->addError($e->getMessage());
} catch (ApiException $e) {
$app['monolog']->addError($e->getMessage());
}
$progress->advance();
}
$progress->setMessage('<comment>Search ready</comment>');
$progress->clear();
$progress->display();
$progress->finish();
$output->writeln('');
$output->writeln("<info>Saving data in {$target}</info>");
$phpExcel->setActiveSheetIndex(0);
$writer = PHPExcel_IOFactory::createWriter($phpExcel, 'Excel2007');
$writer->save($target);
$output->writeln("<info>Data saved in {$target}</info>");
break;
default:
$output->writeln('<error>Unrecognized API</error>');
break;
}
} else {
$output->writeln('<error>The target file ' . $source . ' does not exist</error>');
}
/**
* @param ProgressBar $progressHelper
*/
private function iterateFiles(ProgressBar $progressHelper)
{
foreach ($this->getFinder()->getIterator() as $file) {
/** @var \Symfony\Component\Finder\SplFileInfo $file */
if (OutputInterface::VERBOSITY_VERBOSE <= $this->getOutput()->getVerbosity()) {
$progressHelper->clear();
$this->getOutput()->writeln("\r" . $file->getRealPath());
$progressHelper->display();
}
$this->getAnalyzer()->analyze($file);
$progressHelper->advance();
}
}
/**
* For given $contentId in $versionNo updates fields of $fieldTypeIdentifier type.
*
* @param \Symfony\Component\Console\Output\OutputInterface $output
* @param \Symfony\Component\Console\Helper\ProgressBar $progress
* @param int|string $contentId
* @param int $versionNo
* @param string $fieldTypeIdentifier
* @param boolean $dryRun
*/
protected function updateField(OutputInterface $output, ProgressBar $progress, $contentId, $versionNo, $fieldTypeIdentifier, $dryRun)
{
$container = $this->getContainer();
/** @var \eZ\Publish\SPI\FieldType\FieldType $fieldType */
/** @var \eZ\Publish\Core\Persistence\Legacy\Content\FieldValue\Converter $converter */
/** @var \eZ\Publish\Core\Persistence\Legacy\Content\Gateway $gateway */
/** @var \eZ\Publish\SPI\Persistence\Content\Handler $contentHandler */
/** @var \eZ\Publish\API\Repository\ContentService $contentService */
/** @var \eZ\Publish\API\Repository\ContentTypeService $contentTypeService */
$fieldType = $container->get("ezpublish.fieldType.{$fieldTypeIdentifier}");
$converter = $container->get("ezpublish.fieldType.{$fieldTypeIdentifier}.converter");
$gateway = $container->get("ezpublish.persistence.legacy.content.gateway");
$contentHandler = $container->get("ezpublish.spi.persistence.legacy.content.handler");
$contentService = $container->get("ezpublish.api.service.content");
$contentTypeService = $container->get("ezpublish.api.service.content_type");
$content = $contentService->loadContent($contentId, null, $versionNo);
$spiContent = $contentHandler->load($contentId, $versionNo);
$contentType = $contentTypeService->loadContentType($content->contentInfo->contentTypeId);
foreach ($content->getFields() as $field) {
$fieldDefinition = $contentType->getFieldDefinition($field->fieldDefIdentifier);
// Match API field by field type
if ($fieldDefinition->fieldTypeIdentifier != $fieldTypeIdentifier) {
continue;
}
foreach ($spiContent->fields as $spiField) {
// Match SPI field with API field by ID
if ($spiField->id != $field->id) {
continue;
}
// Cast API field value to persistence value
$persistenceValue = $fieldType->toPersistenceValue($field->value);
if ($persistenceValue->sortKey == $spiField->value->sortKey) {
// Assume stored value is correct
break;
}
if (!$dryRun) {
// Create and fill Legacy Storage field value
$storageValue = new StorageFieldValue();
$converter->toStorageValue($persistenceValue, $storageValue);
$gateway->updateField($spiField, $storageValue);
}
$progress->clear();
$output->write("\r");
$output->writeln("Updated Content " . $content->id . ", version " . $content->versionInfo->versionNo . ", field " . $spiField->id . ": '" . $spiField->value->sortKey . "' => '" . $persistenceValue->sortKey . "'");
$output->write("\r");
$progress->display();
// Continue outer loop
break;
}
}
}
/**
* Installation modules.
*
* @param InputInterface $input
* @param OutputInterface $output
* @param array $modules
* @throws BitrixException
* @throws \LogicException
* @throws ModuleException
*/
protected function configureModules(InputInterface $input, OutputInterface $output, array $modules)
{
$app = $this->getApplication();
if ($app->getConfiguration()) {
$app->addCommands(ModuleCommand::getCommands());
if ($app->getBitrixStatus() != \Notamedia\ConsoleJedi\Application\Application::BITRIX_STATUS_COMPLETE) {
throw new BitrixException('Bitrix core is not available');
}
} else {
throw new BitrixException('No configuration loaded');
}
if (!is_array($modules)) {
throw new \LogicException('Incorrect modules configuration');
}
if (!count($modules)) {
return;
}
$bar = new ProgressBar($output, count($modules));
$bar->setRedrawFrequency(1);
$bar->setFormat('verbose');
foreach ($modules as $moduleName) {
$message = "\r" . ' module:load ' . $moduleName . ': ';
try {
if (isset($bar)) {
$bar->setMessage($message);
$bar->display();
}
(new Module($moduleName))->load()->register();
$bar->clear();
$output->writeln($message . '<info>ok</info>');
} catch (ModuleInstallException $e) {
$bar->clear();
$output->writeln($e->getMessage(), OutputInterface::VERBOSITY_VERBOSE);
$output->writeln($message . '<comment>not registered</comment> (install it in admin panel)');
} catch (ModuleException $e) {
$bar->clear();
$output->writeln($e->getMessage(), OutputInterface::VERBOSITY_VERBOSE);
$output->writeln($message . '<error>FAILED</error>');
}
$bar->advance();
}
$bar->finish();
$bar->clear();
$output->write("\r");
}
/**
* @param $file
* @param ProgressBar $progress
*/
private function processGtfs($file, ProgressBar $progress)
{
$mergers = ['agencyMerger' => 'agency.txt', 'routesMerger' => 'routes.txt', 'stopsMerger' => 'stops.txt', 'calendarMerger' => 'calendar.txt', 'calendarDatesMerger' => 'calendar_dates.txt', 'tripsMerger' => 'trips.txt', 'stopTimesMerger' => 'stop_times.txt', 'externalIDsMerger' => 'stop_external_ids.txt'];
if (!is_readable($file)) {
throw new InvalidArgumentException('Cannot read file/s in path "' . $file . '". Aborting.');
}
$zip = new ZipArchiveAdapter($file);
foreach ($mergers as $merger => $subfile) {
$progress->setMessage($subfile, 'gtfs_part');
$progress->display();
$resource = $zip->readStream($subfile);
if ($resource === false || !is_resource($resource['stream'])) {
throw new InvalidArgumentException('Cannot find or read GTFS part "' . $subfile . '" in file "' . $file . '". Aborting.');
}
$items = $this->{$merger}->merge($resource['stream']);
$this->gtfsWriter->append($subfile, $items);
}
}
public function buildLoggerClosure(OutputInterface $output, $total)
{
$progress = null;
return function ($message, $increment = null) use(&$progress, $output, $total) {
if (null === $progress) {
$progress = new ProgressBar($output, $total);
$progress->setFormat('debug');
$progress->start();
}
$progress->setMessage(sprintf('<info>%s</info>', $message));
$increment == null ? $progress->display() : $progress->advance($increment);
};
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$path = $this->getApplication()->getRefDir();
$iterator = new \DirectoryIterator($path);
$suffix = '.extensions.json';
foreach ($iterator as $file) {
if (fnmatch('*' . $suffix, $file->getPathName())) {
$className = str_replace($suffix, '', basename($file));
$extName = strtolower($className);
$this->extensions[] = $extName;
}
}
$extension = trim($input->getArgument('extension'));
$extension = strtolower($extension);
if (empty($extension)) {
$extensions = $this->extensions;
} else {
if (!in_array($extension, $this->extensions)) {
$output->writeln(sprintf('<error>Extension %s does not exist.</error>', $extension));
return;
}
$extensions = array($extension);
}
// do a DB backup first
$command = $this->getApplication()->find('db:backup');
$arguments = array('command' => 'db:backup');
$input = new ArrayInput($arguments);
$returnCode = $command->run($input, $output);
if ($returnCode !== 0) {
$output->writeln('<error>DB backup not performed</error>');
return;
}
// then delete current DB before to init a new copy again
unlink($this->getApplication()->getDbFilename());
$pdo = new \PDO('sqlite:' . $this->getApplication()->getDbFilename());
$ref = new ReferenceCollection($pdo);
$max = count($extensions);
$progress = new ProgressBar($output, $max);
$progress->setFormat(' %percent:3s%% %elapsed:6s% %memory:6s% %message%');
$progress->setMessage('');
$progress->start();
foreach ($extensions as $refName) {
$pdo->beginTransaction();
$ext = 'extensions';
$progress->setMessage(sprintf("Building %s (%s)", $ext, $refName));
$progress->display();
$data = $this->readJsonFile($refName, $ext, '');
$ref->addExtension($data);
$ext = 'releases';
$data = $this->readData($refName, $ext);
foreach ($data as $rec) {
$ref->addRelease($rec);
}
$ext = 'interfaces';
$data = $this->readData($refName, $ext);
foreach ($data as $rec) {
$ref->addInterface($rec);
}
$ext = 'classes';
$data = $this->readData($refName, $ext);
foreach ($data as $rec) {
$ref->addClass($rec);
}
$ext = 'functions';
$data = $this->readData($refName, $ext);
foreach ($data as $rec) {
$ref->addFunction($rec);
}
$ext = 'constants';
$data = $this->readData($refName, $ext);
foreach ($data as $rec) {
$ref->addConstant($rec);
}
$ext = 'iniEntries';
$data = $this->readData($refName, $ext);
foreach ($data as $rec) {
$ref->addIniEntry($rec);
}
$ext = 'const';
$data = $this->readData($refName, $ext);
foreach ($data as $rec) {
$ref->addClassConstant($rec);
}
$ext = 'methods';
$data = $this->readData($refName, $ext);
foreach ($data as $rec) {
$ref->addMethod($rec);
}
$pdo->commit();
$progress->advance();
}
$ref->addVersion(array('build_string' => date('M d Y H:i:s T'), 'build_date' => date('YmdHis')));
$progress->setMessage('Database is built');
$progress->display();
$progress->finish();
$output->writeln('');
}
/**
* @param Driver $targetDriver
* @param int $currentVersion
* @param int $latestVersion
*
* @return void
*/
private function runUpdates(Driver $targetDriver, int $currentVersion, int $latestVersion)
{
$progress = new ProgressBar($this->output, $latestVersion - $currentVersion);
$progress->setFormat('%message% [%bar%] %version% %percent:3s%%');
$progress->setMessage('Applying version');
$progress->setMessage('', 'version');
$progress->start();
for ($version = $currentVersion + 1; $version <= $latestVersion; $version++) {
$progress->setMessage("{$version}/{$latestVersion}", 'version');
$progress->display();
$sqlFilePath = $this->getSqlPath() . '/' . $version . '.sql';
$this->startUpdate($targetDriver, $version);
$this->importSqlFile($targetDriver->getPdo(), $sqlFilePath);
$this->endUpdate($targetDriver, $version);
$progress->advance();
}
$progress->finish();
$this->output->writeln('');
}
/**
* @param OutputInterface $output
* @param ItemId[] $itemIds
* @param bool $force
*/
private function executeForItemIds(OutputInterface $output, array $itemIds, $force)
{
$itemLookup = $this->wikibaseFactory->newItemLookup();
$processedItemIdStrings = $this->getProcessedItemIdStrings();
$loopCounter = 0;
/** @var FormatterHelper $formatter */
$formatter = $this->getHelper('formatter');
foreach ($itemIds as $itemId) {
$loopCounter++;
$itemIdString = $itemId->getSerialization();
$output->writeln('----------------------------------------------------');
if ($loopCounter % 10 != 0) {
$processedItemIdStrings = $this->getProcessedItemIdStrings();
}
if (!$force && in_array($itemId->getSerialization(), $processedItemIdStrings)) {
$output->writeln($formatter->formatSection($itemIdString, 'Already processed'));
continue;
}
try {
$output->writeln($formatter->formatSection($itemIdString, 'Loading Item'));
$item = $itemLookup->getItemForId($itemId);
} catch (ItemLookupException $e) {
$output->writeln($formatter->formatSection($itemIdString, 'Failed to load item (exception)', 'error'));
continue;
}
if ($item === null) {
$output->writeln($formatter->formatSection($itemIdString, 'Failed to load item (null)', 'error'));
continue;
}
// Get the item types..
$types = array();
foreach ($item->getStatements()->getByPropertyId(new PropertyId('P31'))->toArray() as $instanceStatement) {
$mainSnak = $instanceStatement->getMainSnak();
if ($mainSnak instanceof PropertyValueSnak) {
/** @var EntityIdValue $instanceItemIdValue */
$instanceItemIdValue = $mainSnak->getDataValue();
$idSerialization = $instanceItemIdValue->getEntityId()->getSerialization();
if (array_key_exists($idSerialization, $this->instanceMap)) {
$types[] = $this->instanceMap[$idSerialization];
}
}
}
if (empty($types)) {
$output->writeln($formatter->formatSection($itemIdString, 'Didn\\t find any useful instance of statements', 'comment'));
continue;
}
// Note: only load Wikipedias
$siteLinkList = DataModelUtils::getSitelinksWiteSiteIdSuffix($item->getSiteLinkList(), 'wiki');
$output->writeln($formatter->formatSection($itemIdString, $siteLinkList->count() . ' Wikipedia pages to request'));
$parseProgressBar = new ProgressBar($output, $siteLinkList->count());
$parseProgressBar->display();
/** @var PromiseInterface[] $parsePromises */
$parsePromises = array();
foreach ($siteLinkList->getIterator() as $siteLink) {
$siteId = $siteLink->getSiteId();
$pageName = $item->getSiteLinkList()->getBySiteId($siteId)->getPageName();
$sourceMwFactory = $this->wmFactoryFactory->getFactory($siteId);
$sourceParser = $sourceMwFactory->newParser();
$pageIdentifier = new PageIdentifier(new Title($pageName));
$parsePromises[$siteId] = $sourceParser->parsePageAsync($pageIdentifier);
$parseProgressBar->advance();
}
$links = array();
foreach ($parsePromises as $siteId => $promise) {
try {
$parseResult = $promise->wait();
if (array_key_exists('externallinks', $parseResult)) {
foreach ($parseResult['externallinks'] as $externalLink) {
// Ignore archive.org links
if (strstr($externalLink, 'archive.org') === false) {
$links[] = $this->normalizeExternalLink($externalLink);
}
}
}
} catch (Exception $e) {
$parseProgressBar->clear();
$output->writeln($formatter->formatSection($itemIdString, $e->getMessage(), 'error'));
$parseProgressBar->display();
// Ignore failed requests
}
}
$parseProgressBar->finish();
$output->writeln('');
$links = array_unique($links);
shuffle($links);
/** @var Request[] $linkRequests */
$linkRequests = array();
foreach ($links as $link) {
$linkRequests[] = new Request('GET', $link, array('allow_redirects' => array('track_redirects' => true), 'connect_timeout' => 3.14, 'timeout' => 10));
}
$output->writeln($formatter->formatSection($itemIdString, count($linkRequests) . ' External links to (download, action)'));
if (empty($linkRequests)) {
continue;
}
// Make a bunch of requests and act on the responses
$referencesAddedToItem = 0;
$externalLinkProgressBar = new ProgressBar($output, count($linkRequests) * 2);
$externalLinkProgressBar->display();
$pool = new Pool($this->externalLinkClient, $linkRequests, array('fulfilled' => function ($response) use($externalLinkProgressBar, $item, $types, $referencesAddedToItem, $output) {
$externalLinkProgressBar->advance();
// 1st advance point
if ($response instanceof ResponseInterface) {
$link = $response->getHeaderLine('X-GUZZLE-EFFECTIVE-URL');
$html = $response->getBody();
$referencesAddedFromLink = 0;
foreach ($this->microDataExtractor->extract($html) as $microData) {
foreach ($types as $type) {
if ($microData->hasType($type) && array_key_exists($type, $this->referencerMap)) {
foreach ($this->referencerMap[$type] as $referencer) {
/** @var Referencer $referencer */
$addedReferences = $referencer->addReferences($microData, $item, $link);
$referencesAddedToItem += $addedReferences;
$referencesAddedFromLink += $addedReferences;
}
}
}
}
if ($referencesAddedFromLink > 0) {
$externalLinkProgressBar->clear();
$output->write("\r");
$output->writeln($referencesAddedFromLink . ' reference(s) added from ' . urldecode($link));
$externalLinkProgressBar->display();
}
}
$externalLinkProgressBar->advance();
// 2nd advance point
}, 'rejected' => function () use($externalLinkProgressBar) {
// TODO add this to some kind of verbose log?
$externalLinkProgressBar->advance();
// 1st advance point
}));
$pool->promise()->wait();
$externalLinkProgressBar->finish();
$output->writeln('');
$output->writeln($formatter->formatSection($itemIdString, $referencesAddedToItem . ' References added'));
$this->markIdAsProcessed($itemId);
}
}
public function reDrawProgressBar()
{
if ($this->progress !== null && $this->progress->getStartTime() !== null && $this->progress->getProgress() !== $this->progress->getMaxSteps() && $this->progresseBar === true) {
$this->progress->display();
}
}
/**
* Stores data into the filesystem and returns stored stats
*
* @param BackupJob $job
* @param JobStats $jobStats
* @param OutputInterface $output
* @return array
* @author Daniel Wendlandt
*/
private function storeData(BackupJob $job, JobStats $jobStats, OutputInterface $output)
{
$memoryAtSection = memory_get_usage();
$timeStartSection = microtime(true);
$output->writeln('<info>*** Starting with data storing ***</info>' . PHP_EOL);
$docCount = $this->elastic->getDocCountByIndexType($job->getHost(), $job->getPort());
$storedStats = array();
/** @var Index $index */
foreach ($job->getMappings()->getIndices() as $index) {
if (0 === $docCount->getDocCount($index->getName())) {
continue;
}
/** @var Type $type */
foreach ($index->getTypes() as $type) {
$docsInType = $docCount->getDocCount($index->getName(), $type->getName());
if (0 === $docsInType) {
continue;
}
$scrollId = $this->elastic->createScrollSearch($index->getName(), $type->getName(), $job->getHost(), $job->getPort());
$storedStats[$index->getName()][$type->getName()]['aggregatedNumberOfDocs'] = $docsInType;
$storedStats[$index->getName()][$type->getName()]['storedNumberOfDocs'] = 0;
$output->writeln('<comment>Store Data for: ' . $index->getName() . '/' . $type->getName() . '</comment>');
$progress = new ProgressBar($output, $docsInType);
$progress->setFormat('debug');
$progress->display();
$progress->start();
while (true) {
$data = $this->elastic->getScrollSearchData($scrollId, $job->getHost(), $job->getPort());
if (0 === count($data['hits'])) {
break;
}
$scrollId = $data['scrollId'];
$storedDocs = $this->filesystem->storeData($job->getPath(), $index->getName(), $type->getName(), $data['hits']);
$storedStats[$index->getName()][$type->getName()]['storedNumberOfDocs'] += $storedDocs;
$progress->advance($storedDocs);
}
$progress->finish();
$output->writeln(PHP_EOL);
}
}
$jobStats->setStoreData(microtime(true) - $timeStartSection, memory_get_usage(), memory_get_usage() - $memoryAtSection, array('stats' => $storedStats));
return $storedStats;
}
/**
* Show progress bar and run the loop
*
* @param string $name
* @param int $total
* @param int $stepSize
* @param \Closure $callback
*/
protected function _progressBar($name, $total, $stepSize, $callback)
{
$this->_('Current progress of ' . $name . ' (Wait! or `Ctrl+C` to cancel):');
$progressBar = new ProgressBar($this->_out, $total);
$progressBar->display();
$progressBar->setRedrawFrequency(1);
for ($currentStep = 0; $currentStep <= $total; $currentStep += $stepSize) {
$callbackResult = $callback($currentStep, $stepSize);
if ($callbackResult === false) {
break;
}
$progressBar->setProgress($currentStep);
}
$progressBar->finish();
$this->_('');
// Progress bar hack for rendering
}
public function reDrawProgressBar()
{
if ($this->progress !== null && $this->progress->getStartTime() !== null) {
$this->progress->display();
}
}
public function testPercentNotHundredBeforeComplete()
{
$bar = new ProgressBar($output = $this->getOutputStream(), 200);
$bar->start();
$bar->display();
$bar->advance(199);
$bar->advance();
rewind($output->getStream());
$this->assertEquals($this->generateOutput(' 0/200 [>---------------------------] 0%') . $this->generateOutput(' 0/200 [>---------------------------] 0%') . $this->generateOutput(' 199/200 [===========================>] 99%') . $this->generateOutput(' 200/200 [============================] 100%'), stream_get_contents($output->getStream()));
}
/**
* Log warning while progress helper is running.
*
* @param \Symfony\Component\Console\Output\OutputInterface $output
* @param \Symfony\Component\Console\Helper\ProgressBar $progress
* @param $message
*/
private function logWarning(OutputInterface $output, ProgressBar $progress, $message)
{
$progress->clear();
$this->logger->warning($message);
$progress->display();
}
/**
* Executes all prepared batches of requests.
* @param InputInterface $input
* @param OutputInterface $output
* @return array
*/
protected function executeBatches(InputInterface $input, OutputInterface $output)
{
$totalResult = array();
if (!$this->_options['silent']) {
$output->writeln('<info>Found ' . count($this->_batches) . ' sets to process.</info>');
}
$setCount = count($this->_batches);
$bi = 1;
// batch number
foreach ($this->_batches as $set) {
// sitemaps
$meta = $set['metadata'];
$batchesCount = count($set['requests']);
$setResults = array();
if (!$this->_options['silent']) {
$progress = new ProgressBar($output, $batchesCount);
$progress->setFormat('<info> %message% </info>' . PHP_EOL . '%current%/%max% [%bar%] <comment> %percent:3s%% - %elapsed:6s%/%estimated:-6s% </comment>');
$progress->setMessage('Now executing batch: ' . $bi . '/' . $setCount);
$progress->start();
}
foreach ($set['requests'] as $batch) {
// the batches of requests, singular or plural
$batchResult = array();
foreach ($batch as $request) {
$response = $this->simpleCurl($request);
$response['comparison_key'] = strtr($response['url'], array(rtrim($this->_options['compare-url'], "/") => '', rtrim($this->_options['current-url'], "/") => ''));
array_push($batchResult, $response);
}
array_push($setResults, $batchResult);
if (!$this->_options['silent']) {
$progress->clear();
$progress->setMessage($this->parseStatusMessage($batchResult));
$progress->display();
$progress->advance();
}
}
if (!$this->_options['silent']) {
$progress->setMessage('Task is finished');
$progress->finish();
}
array_push($totalResult, $setResults);
$bi++;
}
return $totalResult;
}
/**
* Fixes always available flag on fields of Content $contentId in $versionNo.
*
* @param \Symfony\Component\Console\Output\OutputInterface $output
* @param \Symfony\Component\Console\Helper\ProgressBar $progress
* @param int $contentId
* @param int $versionNo
* @param int $mainLanguageId
* @param bool $dryRun
*/
protected function updateAlwaysAvailableFlag(OutputInterface $output, ProgressBar $progress, $contentId, $versionNo, $mainLanguageId, $dryRun)
{
if ($dryRun) {
goto output;
}
/** @var \eZ\Publish\Core\Persistence\Database\DatabaseHandler $databaseHandler */
$databaseHandler = $this->getContainer()->get('ezpublish.connection');
// Remove always available flag from all fields not in main language
/** @var $query \eZ\Publish\Core\Persistence\Database\UpdateQuery */
$query = $databaseHandler->createUpdateQuery();
$query->update($databaseHandler->quoteTable('ezcontentobject_attribute'))->set($databaseHandler->quoteColumn('language_id'), $query->expr->bitAnd($databaseHandler->quoteColumn('language_id'), -2))->where($query->expr->lAnd($query->expr->eq($databaseHandler->quoteColumn('contentobject_id'), $query->bindValue($contentId, null, PDO::PARAM_INT)), $query->expr->eq($databaseHandler->quoteColumn('version'), $query->bindValue($versionNo, null, PDO::PARAM_INT)), $query->expr->eq($query->expr->bitAnd($databaseHandler->quoteColumn('language_id'), $query->bindValue($mainLanguageId, null, PDO::PARAM_INT)), $query->bindValue(0, null, PDO::PARAM_INT))))->prepare()->execute();
// Set always available flag on fields in main language
$query = $databaseHandler->createUpdateQuery();
$query->update($databaseHandler->quoteTable('ezcontentobject_attribute'))->set($databaseHandler->quoteColumn('language_id'), $query->expr->bitOr($databaseHandler->quoteColumn('language_id'), 1))->where($query->expr->lAnd($query->expr->eq($databaseHandler->quoteColumn('contentobject_id'), $query->bindValue($contentId, null, PDO::PARAM_INT)), $query->expr->eq($databaseHandler->quoteColumn('version'), $query->bindValue($versionNo, null, PDO::PARAM_INT)), $query->expr->gt($query->expr->bitAnd($databaseHandler->quoteColumn('language_id'), $query->bindValue($mainLanguageId, null, PDO::PARAM_INT)), $query->bindValue(0, null, PDO::PARAM_INT))))->prepare()->execute();
output:
$progress->clear();
$output->write("\r");
$output->writeln("Updated fields for Content '{$contentId}' in version '{$versionNo}'");
$output->write("\r");
$progress->display();
}
