protected function processGroup($group, $lines)
{
// Do nothing if there is no line
if (empty($lines) || is_string($lines)) {
return;
}
array_set($this->missing, $group, array());
$this->line("Processing {$group} group...");
$this->progress->start($this->output, count($lines, COUNT_RECURSIVE));
// Iterate over passed lines
foreach ($lines as $line => $translation) {
// Run recursive if translation is an array
if (is_array($translation)) {
$this->processGroup($group . '.' . $line, $line);
} else {
if ($this->manager->findLineCount($group, $line) == 0) {
array_push($this->missing[$group], $line);
}
$this->progress->advance();
}
}
$this->line(" ");
// Add line break to stop lines from joining
if ($missing = array_get($this->missing, $group)) {
foreach ($missing as $m) {
if ($this->input->getOption('silent') or $confirm = $this->confirm("{$group}.{$m} is missing. Remove? [yes/no]")) {
$this->manager->removeLine($group, $m);
$this->info("Removed {$m} from {$group}");
}
}
}
}
/**
* Execute command
*
* @param InputInterface $input
* @param OutputInterface $output
* @return int|null|void
*/
protected function execute(InputInterface $input, OutputInterface $output)
{
$registry = PlayerRegistry::getDefaultPlayers();
$player1 = $input->getArgument('player-x');
$player2 = $input->getArgument('player-o');
$count = (int) $input->getOption('games');
$progress = new ProgressHelper();
$progress->start($output, $count);
$stats = [PlayerInterface::SYMBOL_X => 0, PlayerInterface::SYMBOL_O => 0, 'Draw' => 0];
for ($i = 0; $i < $count; $i++) {
$game = new Game();
$game->addPlayer($registry->get($player1, PlayerInterface::SYMBOL_X));
$game->addPlayer($registry->get($player2), PlayerInterface::SYMBOL_O);
$winner = $game->autoPlay();
$stats[$winner ? $winner : 'Draw']++;
$progress->advance();
}
$progress->finish();
$output->writeln('');
$output->writeln('Winning statistics');
$table = new TableHelper();
$table->setHeaders([$player1, $player2, "Draw"]);
$table->addRow(array_values($stats));
$table->render($output);
}
/**
* Executes a raw import.
*
* @param Manager $manager
* @param string $filename
* @param OutputInterface $output
* @param int $bulkSize
*/
protected function executeRawImport(Manager $manager, $filename, OutputInterface $output, $bulkSize)
{
$reader = $this->getReader($manager, $filename, false);
if (class_exists('\\Symfony\\Component\\Console\\Helper\\ProgressBar')) {
$progress = new ProgressBar($output, $reader->count());
$progress->setRedrawFrequency(100);
$progress->start();
} else {
$progress = new ProgressHelper();
$progress->setRedrawFrequency(100);
$progress->start($output, $reader->count());
}
foreach ($reader as $key => $document) {
$data = $document['_source'];
$data['_id'] = $document['_id'];
$manager->getConnection()->bulk('index', $document['_type'], $data);
if (($key + 1) % $bulkSize == 0) {
$manager->commit();
}
$progress->advance();
}
if (($key + 1) % $bulkSize != 0) {
$manager->commit();
}
$progress->finish();
$output->writeln('');
}
public function execute(Request $request, $count, OutputInterface $out)
{
$client = new \Elasticsearch\Client(['hosts' => [$request->getHost()]]);
$response = new Response();
if ($this->clearCache) {
$this->clearCache($client, $out);
}
$helper = new ProgressHelper();
$helper->start($out, $count);
$start = microtime(true);
for ($i = 0; $i < $count; $i++) {
try {
$response->addSuccess($client->search($request->getParameters()));
} catch (\Exception $ex) {
$response->addFailure();
var_dump($ex->getMessage());
exit;
}
$helper->advance();
}
$helper->finish();
$response->setExternalTime(microtime(true) - $start);
try {
$response->setStats($client->indices()->stats([['index' => $request->getIndex()], 'groups' => [$request->getStatId()]]));
} catch (\Exception $ex) {
// nothing to do here
}
return $response;
}
public function testPercentNotHundredBeforeComplete()
{
$progress = new ProgressHelper();
$progress->start($output = $this->getOutputStream(), 200);
$progress->display();
$progress->advance(199);
$progress->advance();
rewind($output->getStream());
$this->assertEquals($this->generateOutput(' 0/200 [>---------------------------] 0%') . $this->generateOutput(' 199/200 [===========================>] 99%') . $this->generateOutput(' 200/200 [============================] 100%'), stream_get_contents($output->getStream()));
}
public function testPercent()
{
$progress = new ProgressHelper();
$progress->start($output = $this->getOutputStream(), 50);
$progress->display();
$progress->advance();
$progress->advance();
rewind($output->getStream());
$this->assertEquals($this->generateOutput(' 0/50 [>---------------------------] 0%') . $this->generateOutput(' 1/50 [>---------------------------] 2%') . $this->generateOutput(' 2/50 [=>--------------------------] 4%'), stream_get_contents($output->getStream()));
}
protected function finalizeAllDumps(array $dumpers)
{
// finalize and save dumped files
$this->stopwatch->start('finalizing_files', 'generate_dumps');
$this->progress->start($this->output, count($dumpers));
$this->progress->setBarCharacter('<comment>=</comment>');
$files = parent::finalizeAllDumps($dumpers);
$this->progress->finish();
$this->stopwatch->stop('finalizing_files');
return $files;
}
public function run()
{
foreach ($this->processes as $process) {
/** @var $process Process **/
$process->setIdleTimeout($this->idleTimeout);
$process->setTimeout($this->timeout);
$process->start();
$this->printTaskInfo($process->getCommandLine());
}
$progress = new ProgressHelper();
$progress->setFormat(" <fg=white;bg=cyan;options=bold>[" . get_class($this) . "]</fg=white;bg=cyan;options=bold> Processes: %current%/%max% [%bar%] %percent%%");
$progress->start($this->getOutput(), count($this->processes));
$running = $this->processes;
$progress->display();
$started = microtime(true);
while (true) {
foreach ($running as $k => $process) {
try {
$process->checkTimeout();
} catch (ProcessTimedOutException $e) {
}
if (!$process->isRunning()) {
$progress->advance();
if ($this->isPrinted) {
$this->getOutput()->writeln("");
$this->printTaskInfo("Output for <fg=white;bg=magenta> " . $process->getCommandLine() . " </fg=white;bg=magenta>");
$this->getOutput()->writeln($process->getOutput(), OutputInterface::OUTPUT_RAW);
if ($process->getErrorOutput()) {
$this->getOutput()->writeln("<error>" . $process->getErrorOutput() . "</error>");
}
}
unset($running[$k]);
}
}
if (empty($running)) {
break;
}
usleep(1000);
}
$this->getOutput()->writeln("");
$taken = number_format(microtime(true) - $started, 2);
$this->printTaskInfo(count($this->processes) . " processes ended in {$taken} s");
$errorMessage = '';
$exitCode = 0;
foreach ($this->processes as $p) {
if ($p->getExitCode() === 0) {
continue;
}
$errorMessage .= "'" . $p->getCommandLine() . "' exited with code " . $p->getExitCode() . " \n";
$exitCode = max($exitCode, $p->getExitCode());
}
return new Result($this, $exitCode, $errorMessage);
}
/**
* Prefix url of document with current resourcelocator prefix.
*
* @param IndexRebuildEvent $event
*/
public function onIndexRebuild(IndexRebuildEvent $event)
{
$output = $event->getOutput();
$filter = $event->getFilter();
$output->writeln('<info>Rebuilding content index</info>');
$typeMap = $this->baseMetadataFactory->getPhpcrTypeMap();
$phpcrTypes = [];
foreach ($typeMap as $type) {
$phpcrType = $type['phpcr_type'];
if ($phpcrType !== 'sulu:path') {
$phpcrTypes[] = sprintf('[jcr:mixinTypes] = "%s"', $phpcrType);
}
}
$condition = implode(' or ', $phpcrTypes);
// TODO: We cannot select all contents via. the parent type, see: https://github.com/jackalope/jackalope-doctrine-dbal/issues/217
$query = $this->documentManager->createQuery('SELECT * FROM [nt:unstructured] AS a WHERE ' . $condition);
$count = [];
$documents = $query->execute();
$progress = new ProgressHelper();
$progress->start($output, count($documents));
foreach ($documents as $document) {
if ($document instanceof SecurityBehavior && !empty($document->getPermissions())) {
$progress->advance();
continue;
}
$locales = $this->inspector->getLocales($document);
foreach ($locales as $locale) {
try {
$this->documentManager->find($document->getUuid(), $locale);
$documentClass = get_class($document);
if ($filter && !preg_match('{' . $filter . '}', $documentClass)) {
continue;
}
$this->searchManager->index($document, $locale);
if (!isset($count[$documentClass])) {
$count[$documentClass] = 0;
}
++$count[$documentClass];
} catch (\Exception $e) {
$output->writeln(sprintf('<error>Error indexing or de-indexing page (path: %s locale: %s)</error>: %s', $this->inspector->getPath($document), $locale, $e->getMessage()));
}
}
$progress->advance();
}
$output->writeln('');
foreach ($count as $className => $count) {
if ($count == 0) {
continue;
}
$output->writeln(sprintf('<comment>Content</comment>: %s <info>%s</info> indexed', $className, $count));
}
}
/**
* @param EventDispatcherInterface $dispatcher
* @param ProgressHelper $progress
* @param OutputInterface $output
*/
protected function setupProgressListeners(EventDispatcherInterface $dispatcher, ProgressHelper $progress, OutputInterface $output)
{
$dispatcher->addListener(IoEvents::PRE_EXPORT_FEED, function (ExportFeedEvent $event) use($progress, $output) {
$output->writeln(sprintf('Exporting feed for <info>%s</info> to <info>%s</info>', $event->getType()->getName(), $event->getFile()));
$progress->start($output, $event->getTotal());
});
$dispatcher->addListener(IoEvents::POST_EXPORT_ITEM, function () use($progress) {
$progress->advance(1);
});
$dispatcher->addListener(IoEvents::POST_EXPORT_FEED, function () use($progress) {
$progress->finish();
});
}
public function testCustomizations()
{
$progress = new ProgressHelper();
$progress->setBarWidth(10);
$progress->setBarCharacter('_');
$progress->setEmptyBarCharacter(' ');
$progress->setProgressCharacter('/');
$progress->setFormat(' %current%/%max% [%bar%] %percent%%');
$progress->start($output = $this->getOutputStream(), 10);
$progress->advance();
rewind($output->getStream());
$this->assertEquals($this->generateOutput(' 1/10 [_/ ] 10%'), stream_get_contents($output->getStream()));
}
/**
* Process list of time entries.
*
* @param $entries
*/
function processTimeEntries($entries)
{
$process = array();
$table = new Table($this->output);
$table->setHeaders(array('Issue', 'Issue title', 'Description', 'Duration', 'Activity', 'Status'));
$defaultActivity = $this->getDefaultRedmineActivity();
// Get the items to process.
foreach ($entries as $entry) {
$activity_type = $this->getRedmineActivityFromTogglEntry($entry);
// Get issue number from description.
if ($issue_id = $this->getIssueNumberFromTimeEntry($entry)) {
// Check if the entry is already synced.
if ($this->isTimeEntrySynced($entry)) {
$table->addRow(array($issue_id, $this->getRedmineIssueTitle($issue_id, '<warning>Issue is not available anymore.</warning>'), $entry['description'], number_format($entry['duration'] / 60 / 60, 2), $activity_type ? $activity_type->name : '', '<info>SYNCED</info>'));
} elseif (!$this->isIssueNumberValid($issue_id)) {
$table->addRow(array($issue_id, '', $entry['description'], number_format($entry['duration'] / 60 / 60, 2), $activity_type ? $activity_type->name : '', '<error>Given issue not available.</error>'));
} elseif ($activity_type || $defaultActivity) {
$table->addRow(array($issue_id, $this->getRedmineIssueTitle($issue_id), $entry['description'], number_format($entry['duration'] / 60 / 60, 2), $activity_type ? $activity_type->name : sprintf('[ %s ]', $defaultActivity->name), '<comment>unsynced</comment>'));
// Set item to be process.
$process[] = array('issue' => $issue_id, 'entry' => $entry, 'activity' => $activity_type ? $activity_type : $defaultActivity);
} else {
$table->addRow(array($issue_id, $this->getRedmineIssueTitle($issue_id), $entry['description'], number_format($entry['duration'] / 60 / 60, 2), '', '<error>no activity</error>'));
}
} else {
$table->addRow(array(' - ', '', $entry['description'], number_format($entry['duration'] / 60 / 60, 2), $activity_type->name, '<error>No Issue ID found</error>'));
}
}
$table->render();
// Simply proceed if no items are to be processed.
if (empty($process)) {
$this->output->writeln('<info>All entries synced</info>');
return;
}
// Confirm before we really process.
if (!$this->question->ask($this->input, $this->output, new ConfirmationQuestion(sprintf('<question> %d entries not synced. Process now? [y] </question>', count($process)), false))) {
$this->output->writeln('<error>Sync aborted.</error>');
return;
}
// Process each item.
$this->progress->start($this->output, count($process));
foreach ($process as $processData) {
$this->syncTimeEntry($processData['entry'], $processData['issue'], $processData['activity']);
$this->progress->advance();
}
$this->progress->finish();
}
/**
* Begin scheme generation.
*
* @return void
*/
protected function generate()
{
// Output purpose.
$this->output->writeln('Generating colour schemes...');
// Get the total count by multiplying patterns by themes.
$count = count($this->patterns) * count($this->themes);
// Create the progress meter.
$this->progress->start($this->output, $count);
// Iterate patterns.
foreach ($this->patterns as $pattern) {
// Iterate themes.
foreach ($this->themes as $theme) {
// Attempt to render a pattern and theme combo.
$this->render($pattern, $theme);
}
}
// Show completion.
$this->progress->finish();
}
/**
* Prefix url of document with current resourcelocator prefix.
*
* @param IndexRebuildEvent $event
*/
public function onIndexRebuild(IndexRebuildEvent $event)
{
$output = $event->getOutput();
$purge = $event->getPurge();
$filter = $event->getFilter();
$output->writeln('<info>Rebuilding content index</info>');
// TODO: We cannot select all contents via. the parent type, see: https://github.com/jackalope/jackalope-doctrine-dbal/issues/217
$query = $this->documentManager->createQuery('SELECT * FROM [nt:unstructured] AS a WHERE [jcr:mixinTypes] = "sulu:page" or [jcr:mixinTypes] = "sulu:snippet"');
$count = [];
if ($purge) {
$this->purgeContentIndexes($output);
}
$documents = $query->execute();
$progress = new ProgressHelper();
$progress->start($output, count($documents));
foreach ($documents as $document) {
$locales = $this->inspector->getLocales($document);
foreach ($locales as $locale) {
try {
$this->documentManager->find($document->getUuid(), $locale);
$documentClass = get_class($document);
if ($filter && !preg_match('{' . $filter . '}', $documentClass)) {
continue;
}
$this->searchManager->index($document, $locale);
if (!isset($count[$documentClass])) {
$count[$documentClass] = 0;
}
++$count[$documentClass];
} catch (\Exception $e) {
$output->writeln(sprintf('<error>Error indexing or de-indexing page (path: %s locale: %s)</error>: %s', $this->inspector->getPath($document), $locale, $e->getMessage()));
}
}
$progress->advance();
}
$output->writeln('');
foreach ($count as $className => $count) {
if ($count == 0) {
continue;
}
$output->writeln(sprintf('<comment>Content</comment>: %s <info>%s</info> indexed', $className, $count));
}
}
/**
* @inheritdoc
*/
public function execute(ResultCollection $collection, ResultCollection $aggregatedResults)
{
$files = $this->finder->find($this->path);
if (0 == sizeof($files, COUNT_NORMAL)) {
throw new \LogicException('No file found');
}
$progress = new ProgressHelper();
$progress->start($this->output, sizeof($files, COUNT_NORMAL));
// tools
$classMap = new ClassMap();
$tokenizer = new Tokenizer();
$syntaxChecker = new SyntaxChecker();
$fileAnalyzer = new FileAnalyzer($this->output, $this->withOOP, new Extractor($tokenizer), new \Hal\Metrics\Complexity\Text\Halstead\Halstead($tokenizer, new \Hal\Component\Token\TokenType()), new \Hal\Metrics\Complexity\Text\Length\Loc($tokenizer), new \Hal\Metrics\Design\Component\MaintenabilityIndex\MaintenabilityIndex($tokenizer), new \Hal\Metrics\Complexity\Component\McCabe\McCabe($tokenizer), new \Hal\Metrics\Complexity\Component\Myer\Myer($tokenizer), $classMap);
foreach ($files as $k => $filename) {
$progress->advance();
// Integrity
if (!$syntaxChecker->isCorrect($filename)) {
$this->output->writeln(sprintf('<error>file %s is not valid and has been skipped</error>', $filename));
unset($files[$k]);
continue;
}
// Analyze
$resultSet = $fileAnalyzer->execute($filename);
$collection->push($resultSet);
}
$progress->clear();
$progress->finish();
if ($this->withOOP) {
// COUPLING (should be done after parsing files)
$this->output->write(str_pad("\rAnalyzing coupling. This will take few minutes...", 80, " "));
$couplingAnalyzer = new CouplingAnalyzer($classMap, $collection);
$couplingAnalyzer->execute($files);
// LCOM (should be done after parsing files)
$this->output->write(str_pad("\rLack of cohesion of method (lcom). This will take few minutes...", 80, " "));
$lcomAnalyzer = new LcomAnalyzer($classMap, $collection);
$lcomAnalyzer->execute($files);
// Card and Agresti (should be done after parsing files)
$this->output->write(str_pad("\rAnalyzing System complexity. This will take few minutes...", 80, " "));
$lcomAnalyzer = new CardAndAgrestiAnalyzer($classMap, $collection);
$lcomAnalyzer->execute($files);
}
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$defaults = array();
if ($input->getOption('msgctxt')) {
$defaults['msgctxt'] = $input->getOption('msgctxt');
}
$this->pot = new Pot($input->getOption('base'), array(), $defaults);
$files = $input->getArgument('files');
if (in_array('-', $files)) {
$files = array_merge($files, explode("\n", stream_get_contents($this->stdin)));
}
$actualFiles = $this->findFiles($files);
if (!$input->getOption('out')) {
foreach ($actualFiles as $file) {
$this->extractFile($file);
}
if ($input->getOption('header')) {
$output->write(file_get_contents($input->getOption('header')));
}
$output->write($this->pot->toString($input));
} else {
$progress = new ProgressHelper();
$progress->start($output, 1 + count($actualFiles));
$progress->advance();
foreach ($actualFiles as $file) {
$this->extractFile($file);
$progress->advance();
}
$content = '';
## If header is supplied and if we're starting a new file.
if ($input->getOption('header')) {
if (!file_exists($input->getOption('out')) || !$input->getOption('append')) {
$content .= file_get_contents($input->getOption('header'));
}
}
$content .= $this->pot->toString($input);
file_put_contents($input->getOption('out'), $content, $input->getOption('append') ? FILE_APPEND : NULL);
$progress->finish();
}
}
public function start($inputDir, $outputDir, OutputInterface $consoleOutput, ProgressHelper $progress)
{
$this->inputDir = $inputDir;
$this->outputDir = $outputDir;
self::$consoleOutput = $consoleOutput;
$inputOutputMappings = $this->createInputOutputMappings();
$availableDataFiles = array();
$progress->start($consoleOutput, count($inputOutputMappings));
foreach ($inputOutputMappings as $textFile => $outputFiles) {
$mappings = $this->readMappingsFromFile($textFile);
$language = $this->getLanguageFromTextFile($textFile);
$this->removeEmptyEnglishMappings($mappings, $language);
$this->makeDataFallbackToEnglish($textFile, $mappings);
$mappingForFiles = $this->splitMap($mappings, $outputFiles);
foreach ($mappingForFiles as $outputFile => $value) {
$this->writeMappingFile($language, $outputFile, $value);
$this->addConfigurationMapping($availableDataFiles, $language, $outputFile);
}
$progress->advance();
}
$this->writeConfigMap($availableDataFiles);
$progress->finish();
}
/**
* Exports es index to provided file.
*
* @param Manager $manager
* @param string $filename
* @param int $chunkSize
* @param OutputInterface $output
*/
public function exportIndex(Manager $manager, $filename, $chunkSize, OutputInterface $output)
{
$types = $manager->getTypesMapping();
$repo = $manager->getRepository($types);
$results = $this->getResults($repo, $chunkSize);
if (class_exists('\\Symfony\\Component\\Console\\Helper\\ProgressBar')) {
$progress = new ProgressBar($output, $results->getTotalCount());
$progress->setRedrawFrequency(100);
$progress->start();
} else {
$progress = new ProgressHelper();
$progress->setRedrawFrequency(100);
$progress->start($output, $results->getTotalCount());
}
$metadata = ['count' => $results->getTotalCount(), 'date' => date(\DateTime::ISO8601)];
$writer = $this->getWriter($this->getFilePath($filename), $metadata);
foreach ($results as $data) {
$writer->push(array_intersect_key($data, array_flip(['_id', '_type', '_source'])));
$progress->advance();
}
$writer->finalize();
$progress->finish();
$output->writeln('');
}
public function testNonDecoratedOutput()
{
$progress = new ProgressHelper();
$progress->start($output = $this->getOutputStream(false));
$progress->advance();
rewind($output->getStream());
$this->assertEquals('', stream_get_contents($output->getStream()));
}
/**
* Do the actual import to our database
* @todo rewrite this, so it can be overwritten - maybe add required import to product?
* @param bool $debug
* @return array|boolean
* @throws Exception\MagmiHasNotBeenSetup
*/
public function import($debug = false)
{
$this->debugMode = $debug;
if ($debug === false) {
if (!$this->magmi instanceof \Magmi_ProductImport_DataPump) {
throw new Exception\MagmiHasNotBeenSetup(sprintf('Magmi has not been setup yet, use setMagmi(%s, %s, %s. %s)', 'Magmi_ProductImport_DataPump $magmi', 'string $profile', 'string $mode', 'Datapump\\Logger Logger $logger'));
}
$this->magmi->beginImportSession($this->profile, $this->mode, $this->logger);
}
$output = array();
if ($this->output instanceof OutputInterface && $debug === false) {
$progress = new ProgressHelper();
$this->output->writeln("\nImport progress");
$progress->start($this->output, count($this->products));
}
foreach ($this->products as $product) {
if ($this->output instanceof OutputInterface && $debug === false) {
$progress->advance();
}
/** @var ProductAbstract $product */
switch ($product->getRequiredData()->getType()) {
case DataAbstract::TYPE_CONFIGURABLE:
/** @var Configurable $product */
if (method_exists($product, 'getSimpleProducts')) {
foreach ($product->getSimpleProducts() as $simple) {
/** @var Simple $simple */
$output[] = $this->inject($simple);
}
}
$output[] = $this->inject($product);
break;
default:
$output[] = $this->inject($product);
break;
}
}
if ($debug === false) {
$this->magmi->endImportSession();
if ($this->output instanceof OutputInterface && $progress instanceof ProgressHelper) {
$progress->finish();
}
} else {
return $output;
}
$this->resetProducts();
return true;
}
/**
* @param int $count The number of founded files
*/
public function start($count)
{
$this->progress->start($this->output, $count);
$this->progress->display();
$this->started = microtime(true);
}
<?php
use Symfony\Component\Console\Helper\ProgressHelper;
// create a new progress bar (50 units)
$progress = new ProgressHelper($output, 50);
// start and displays the progress bar
$progress->start();
$i = 0;
while ($i++ < 50) {
// ... do some work
// advance the progress bar 1 unit
$progress->advance();
// you can also advance the progress bar by more than 1 unit
// $progress->advance(3);
}
// ensure that the progress bar is at 100%
$progress->finish();
public function run()
{
$this->printTaskInfo("creating <info>{$this->filename}</info>");
$this->phar->setSignatureAlgorithm(\Phar::SHA1);
$this->phar->startBuffering();
$this->printTaskInfo('packing ' . count($this->files) . ' files into phar');
$progress = new ProgressHelper();
$progress->start($this->getOutput(), count($this->files));
foreach ($this->files as $path => $content) {
$this->phar->addFromString($path, $content);
$progress->advance();
}
$this->phar->stopBuffering();
$progress->finish();
if ($this->compress and in_array('GZ', \Phar::getSupportedCompression())) {
$this->printTaskInfo($this->filename . " compressed");
$this->phar = $this->phar->compressFiles(\Phar::GZ);
}
$this->printTaskInfo($this->filename . " produced");
return Result::success($this);
}
/**
* Imports the localities of a specific country into the locality
*
* @param string|CountryInterface $country Country
* @param Filter $filter Locality Filter
* @param LoggerInterface $log Import log
*/
public function importCountry($country, Filter $filter = null, LoggerInterface $log = null, ProgressHelper $progressHelper, OutputInterface $outputInterface)
{
$log = $log ?: new NullLogger();
$countryRepository = $this->getCountryRepository();
$managerRegistry = $this->getManagerRegistry();
// Load the country if required
if (!$country instanceof CountryInterface) {
$countryCode = $country;
$country = $countryRepository->getCountry($countryCode);
if (!$country) {
$log->error("Country code '{code}' does not exist in repository {repository}", ['code' => $countryCode, 'repository' => get_class($countryRepository)]);
return [];
}
}
// Retrieve the locality TSV stream
$path = $this->getLocalityDataPath($country, $log);
$log->info("Reading locality data from {path}", ['path' => $path]);
$stream = @fopen($path, 'r');
$separator = "\t";
// As a workaround for the lack of support for fgetcsv() without any
// enclosure value we can use the backspace character as the enclosure
$enclosure = chr(8);
// Check that the stream opened successfully, if it failed to open log
// an error and return execution to the callee
if ($stream === false) {
$log->error("Could not read locality stream for {country} ({code})", ['country' => $country->getName(), 'code' => $country->getCode()]);
return [];
}
// Register that we're now importing localities
$log->info("Importing {country} localities", ['country' => $country->getName()]);
// Iterate over all the localities in the stream
$lineNumber = 0;
$managers = [];
$cities = [];
$states = [];
$substates = [];
$progressHelper->setRedrawFrequency(100);
$progressHelper->start($outputInterface, $this->getLinesCount($stream));
$stream = @fopen($path, 'r');
$repositories = [];
$batchSize = 150;
$i = 0;
// while (false !== $row = fgetcsv($stream, 0, $separator, $enclosure)) {
$content = file_get_contents($path);
$rows = explode("\n", $content);
foreach ($rows as $row) {
$row = explode($separator, $row);
$i++;
$progressHelper->advance();
// Increment the line number
$lineNumber++;
// Log the line number and file
$log->debug("Line {line} of {file}", ['line' => $lineNumber, 'file' => $path]);
// Skip blank rows
if (!$row[0]) {
continue;
}
// Display a warning and skip rows where there are not enough
// columns to generate a locality
if (count($row) < 17) {
$log->warning("Line {line} only has {count} of {expected} columns", ['line' => $lineNumber, 'count' => count($row), 'expected' => 17]);
continue;
}
// User function is called indirectly to allow the createLocality
// method to be solely responsible for the generation of locality
// information
$importedLocality = call_user_func_array([$this, 'createLocality'], $row);
// Determine the locality repository for import
$featureCode = $importedLocality->getFeatureCode();
// Quick FIX because of french changements
/*
if ($featureCode == 'ADM1H') {
$featureCode = 'ADM1';
}
if ($featureCode == 'ADM2H') {
$featureCode = 'ADM2';
}
*/
if (!array_key_exists($featureCode, $repositories)) {
$repositories[$featureCode] = $this->getLocalityRepository($featureCode);
}
$localityRepository = $repositories[$featureCode];
// Skip unsupported geographical features
if (!$localityRepository) {
$log->debug("{name} skipped ({code} feature not supported)", ['code' => $importedLocality->getFeatureCode(), 'name' => $importedLocality->getNameAscii()]);
continue;
}
// Import the specific locality
$locality = $this->importLocality($localityRepository, $importedLocality, $country, $filter, $log);
// Skip non-importable localities
if (!$locality) {
continue;
}
// Determine the manager for the locality
$localityClass = get_class($locality);
if (!isset($managers[$localityClass])) {
$localityManager = $managerRegistry->getManagerForClass($localityClass);
// Ensure the locality manager was loaded
if (!$localityManager) {
$log->error("No object manager registered for {class}", ['class' => get_class($locality)]);
continue;
}
$managers[$localityClass] = $localityManager;
}
$localityManager = $managers[$localityClass];
if (null == $localityManager) {
continue;
}
// Persist the locality
$localityManager->persist($locality);
// $localityManager->clear($locality);
if ($i % $batchSize === 0) {
$localityManager->flush();
// $localityManager->clear();
}
if ($locality instanceof City) {
$cities[] = $locality;
} elseif ($locality instanceof State) {
// || $importedLocality->getFeatureCode() == 'ADM1H'
if ($importedLocality->getFeatureCode() == 'ADM1') {
$states[$locality->getAdmin1Code()] = $locality;
// || $importedLocality->getFeatureCode() == 'ADM2H'
} elseif ($importedLocality->getFeatureCode() == 'ADM2') {
$substates[$locality->getAdmin2Code()] = $locality;
}
}
}
if (isset($localityManager) && null != $localityManager) {
$localityManager->flush();
// $localityManager->clear();
}
$progressHelper->finish();
$outputInterface->writeln("Setting state field on cities...");
$log->info("Saving {country} data", ['country' => $country->getName()]);
// Lets update states on city entities. We cannot do it earlier as desired states may not be loaded on city import
if (count($cities)) {
$cityManager = $managerRegistry->getManagerForClass(get_class($cities[0]));
/** @var $city \JJs\Bundle\GeonamesBundle\Entity\City */
$batchSizeCities = 150;
$iCities = 0;
foreach ($cities as $city) {
$iCities++;
if (isset($states[$city->getAdmin1Code()])) {
$city->setState($states[$city->getAdmin1Code()]);
}
if (isset($substates[$city->getAdmin2Code()])) {
$city->setSubState($substates[$city->getAdmin2Code()]);
}
$cityManager->persist($city);
if ($iCities % $batchSizeCities === 0) {
$cityManager->flush();
// $cityManager->clear();
}
}
$cityManager->flush();
// $cityManager->clear();
}
if (count($substates)) {
reset($substates);
$firstKey = key($substates);
$stateManager = $managerRegistry->getManagerForClass(get_class($substates[$firstKey]));
$batchSizeStates = 150;
$iStates = 0;
foreach ($substates as $substate) {
// var_dump($substate->getNameUtf8() . ' (' . $substate->getAdmin1Code() . '::' . $substate->getAdmin2Code() . ')');
$iStates++;
if (isset($states[$substate->getAdmin1Code()])) {
$state = $states[$substate->getAdmin1Code()];
// var_dump('==>' . $state->getNameUtf8());
$substate->setState($state);
$stateManager->persist($substate);
}
if ($iStates % $batchSizeStates === 0) {
$stateManager->flush();
// $stateManager->clear();
}
}
$stateManager->flush();
// $stateManager->clear();
}
// Flush all managers
foreach ($managers as $manager) {
$manager->flush();
$manager->clear();
}
$log->notice("{code} ({country}) data saved", ['code' => $country->getCode(), 'country' => $country->getName()]);
// Close the locality stream
fclose($stream);
}
/**
* Imports a single file.
*
* @param Package $package the package to import the file into
* @param LoaderInterface $loader
* @param string $path The path to the file
* @param string $filename The filename
* @param bool $backend True to make the file available in the backend
* @param bool $frontend True to make the file available in the frontend
* @param bool $throw If true the methods throws exception if the a file cannot be found
*
* @throws \Exception
* @throws \InvalidArgumentException
* @throws \Symfony\Component\Translation\Exception\NotFoundResourceException
*/
private function importFile($package, $loader, $path, $filename, $backend = true, $frontend = false, $throw = false)
{
try {
$this->output->writeln($filename);
$filePath = $path ? $path . '/' . $filename : $filename;
$file = $loader->load($filePath, $this->locale);
// find the catalogue from this package matching the given locale
$catalogue = null;
$newCatalogue = true;
foreach ($package->getCatalogues() as $packageCatalogue) {
/** @var $packageCatalogue Catalogue */
if ($packageCatalogue->getLocale() === $this->locale) {
$catalogue = $packageCatalogue;
$newCatalogue = false;
}
}
// if no catalogue is found create a new one
if ($newCatalogue === true) {
$catalogue = new Catalogue();
if ($this->locale === $this->defaultLocale) {
$catalogue->setIsDefault(true);
} else {
$catalogue->setIsDefault(false);
}
$catalogue->setPackage($package);
$package->addCatalogue($catalogue);
$catalogue->setLocale($this->locale);
$this->em->persist($catalogue);
$this->em->flush();
}
$allMessages = $file->all()['messages'];
$progress = new ProgressHelper();
$progress->start($this->output, count($allMessages));
// loop through all translation units in the file
foreach ($allMessages as $key => $message) {
// Check if code is already existing in current catalogue
if (!$newCatalogue && ($translate = $catalogue->findTranslation($key))) {
// Update the old translate
$translate->setValue($message);
} else {
// Create new code, if not already existing
$code = $package->findCode($key);
if (!$code) {
$code = new Code();
$code->setPackage($package);
$code->setCode($key);
$code->setBackend($backend);
$code->setFrontend($frontend);
$this->em->persist($code);
$this->em->flush();
}
// Create new translate
$translate = new Translation();
$translate->setCode($code);
$translate->setValue($message);
$translate->setCatalogue($catalogue);
$this->em->persist($translate);
}
$progress->advance();
}
$this->em->flush();
$progress->finish();
} catch (\InvalidArgumentException $e) {
if ($e instanceof NotFoundResourceException) {
if ($throw === true) {
throw $e;
}
} else {
throw $e;
}
}
}
public function testMultiByteSupport()
{
if (!function_exists('mb_strlen') || false === ($encoding = mb_detect_encoding('■'))) {
$this->markTestSkipped('The mbstring extension is needed for multi-byte support');
}
$progress = new ProgressHelper();
$progress->start($output = $this->getOutputStream());
$progress->setBarCharacter('■');
$progress->advance(3);
rewind($output->getStream());
$this->assertEquals($this->generateOutput(' 3 [■■■>------------------------]'), stream_get_contents($output->getStream()));
}
