/**
* Executes a raw import.
*
* @param Manager $manager
* @param string $filename
* @param OutputInterface $output
* @param int $bulkSize
*/
protected function executeRawImport(Manager $manager, $filename, OutputInterface $output, $bulkSize)
{
$reader = $this->getReader($manager, $filename, false);
if (class_exists('\\Symfony\\Component\\Console\\Helper\\ProgressBar')) {
$progress = new ProgressBar($output, $reader->count());
$progress->setRedrawFrequency(100);
$progress->start();
} else {
$progress = new ProgressHelper();
$progress->setRedrawFrequency(100);
$progress->start($output, $reader->count());
}
foreach ($reader as $key => $document) {
$data = $document['_source'];
$data['_id'] = $document['_id'];
$manager->getConnection()->bulk('index', $document['_type'], $data);
if (($key + 1) % $bulkSize == 0) {
$manager->commit();
}
$progress->advance();
}
if (($key + 1) % $bulkSize != 0) {
$manager->commit();
}
$progress->finish();
$output->writeln('');
}
protected function dumpRowsToAllFiles(Config $config, array $dumpers)
{
// generate random data and write row by row
$this->stopwatch->start('generating_rows', 'generate_dumps');
$this->progress->start($this->output, $config->getFakeNumber());
$unit = floor($config->getFakeNumber() / 100);
$this->progress->setRedrawFrequency($unit < 1 ? 1 : $unit);
$this->progress->setBarCharacter('<comment>=</comment>');
$this->output->writeln('Generating <info>' . $config->getFakeNumber() . '</info> rows ...');
parent::dumpRowsToAllFiles($config, $dumpers);
$this->progress->finish();
$this->output->writeln('Finalizing files ...');
$this->stopwatch->stop('generating_rows');
$this->stopwatch->start('finalizing_files', 'generate_dumps');
}
/**
* Exports es index to provided file.
*
* @param Manager $manager
* @param string $filename
* @param int $chunkSize
* @param OutputInterface $output
*/
public function exportIndex(Manager $manager, $filename, $chunkSize, OutputInterface $output)
{
$types = $manager->getTypesMapping();
$repo = $manager->getRepository($types);
$results = $this->getResults($repo, $chunkSize);
if (class_exists('\\Symfony\\Component\\Console\\Helper\\ProgressBar')) {
$progress = new ProgressBar($output, $results->getTotalCount());
$progress->setRedrawFrequency(100);
$progress->start();
} else {
$progress = new ProgressHelper();
$progress->setRedrawFrequency(100);
$progress->start($output, $results->getTotalCount());
}
$metadata = ['count' => $results->getTotalCount(), 'date' => date(\DateTime::ISO8601)];
$writer = $this->getWriter($this->getFilePath($filename), $metadata);
foreach ($results as $data) {
$writer->push(array_intersect_key($data, array_flip(['_id', '_type', '_source'])));
$progress->advance();
}
$writer->finalize();
$progress->finish();
$output->writeln('');
}
/**
* Imports the localities of a specific country into the locality
*
* @param string|CountryInterface $country Country
* @param Filter $filter Locality Filter
* @param LoggerInterface $log Import log
*/
public function importCountry($country, Filter $filter = null, LoggerInterface $log = null, ProgressHelper $progressHelper, OutputInterface $outputInterface)
{
$log = $log ?: new NullLogger();
$countryRepository = $this->getCountryRepository();
$managerRegistry = $this->getManagerRegistry();
// Load the country if required
if (!$country instanceof CountryInterface) {
$countryCode = $country;
$country = $countryRepository->getCountry($countryCode);
if (!$country) {
$log->error("Country code '{code}' does not exist in repository {repository}", ['code' => $countryCode, 'repository' => get_class($countryRepository)]);
return [];
}
}
// Retrieve the locality TSV stream
$path = $this->getLocalityDataPath($country, $log);
$log->info("Reading locality data from {path}", ['path' => $path]);
$stream = @fopen($path, 'r');
$separator = "\t";
// As a workaround for the lack of support for fgetcsv() without any
// enclosure value we can use the backspace character as the enclosure
$enclosure = chr(8);
// Check that the stream opened successfully, if it failed to open log
// an error and return execution to the callee
if ($stream === false) {
$log->error("Could not read locality stream for {country} ({code})", ['country' => $country->getName(), 'code' => $country->getCode()]);
return [];
}
// Register that we're now importing localities
$log->info("Importing {country} localities", ['country' => $country->getName()]);
// Iterate over all the localities in the stream
$lineNumber = 0;
$managers = [];
$cities = [];
$states = [];
$substates = [];
$progressHelper->setRedrawFrequency(100);
$progressHelper->start($outputInterface, $this->getLinesCount($stream));
$stream = @fopen($path, 'r');
$repositories = [];
$batchSize = 150;
$i = 0;
// while (false !== $row = fgetcsv($stream, 0, $separator, $enclosure)) {
$content = file_get_contents($path);
$rows = explode("\n", $content);
foreach ($rows as $row) {
$row = explode($separator, $row);
$i++;
$progressHelper->advance();
// Increment the line number
$lineNumber++;
// Log the line number and file
$log->debug("Line {line} of {file}", ['line' => $lineNumber, 'file' => $path]);
// Skip blank rows
if (!$row[0]) {
continue;
}
// Display a warning and skip rows where there are not enough
// columns to generate a locality
if (count($row) < 17) {
$log->warning("Line {line} only has {count} of {expected} columns", ['line' => $lineNumber, 'count' => count($row), 'expected' => 17]);
continue;
}
// User function is called indirectly to allow the createLocality
// method to be solely responsible for the generation of locality
// information
$importedLocality = call_user_func_array([$this, 'createLocality'], $row);
// Determine the locality repository for import
$featureCode = $importedLocality->getFeatureCode();
// Quick FIX because of french changements
/*
if ($featureCode == 'ADM1H') {
$featureCode = 'ADM1';
}
if ($featureCode == 'ADM2H') {
$featureCode = 'ADM2';
}
*/
if (!array_key_exists($featureCode, $repositories)) {
$repositories[$featureCode] = $this->getLocalityRepository($featureCode);
}
$localityRepository = $repositories[$featureCode];
// Skip unsupported geographical features
if (!$localityRepository) {
$log->debug("{name} skipped ({code} feature not supported)", ['code' => $importedLocality->getFeatureCode(), 'name' => $importedLocality->getNameAscii()]);
continue;
}
// Import the specific locality
$locality = $this->importLocality($localityRepository, $importedLocality, $country, $filter, $log);
// Skip non-importable localities
if (!$locality) {
continue;
}
// Determine the manager for the locality
$localityClass = get_class($locality);
if (!isset($managers[$localityClass])) {
$localityManager = $managerRegistry->getManagerForClass($localityClass);
// Ensure the locality manager was loaded
if (!$localityManager) {
$log->error("No object manager registered for {class}", ['class' => get_class($locality)]);
continue;
}
$managers[$localityClass] = $localityManager;
}
$localityManager = $managers[$localityClass];
if (null == $localityManager) {
continue;
}
// Persist the locality
$localityManager->persist($locality);
// $localityManager->clear($locality);
if ($i % $batchSize === 0) {
$localityManager->flush();
// $localityManager->clear();
}
if ($locality instanceof City) {
$cities[] = $locality;
} elseif ($locality instanceof State) {
// || $importedLocality->getFeatureCode() == 'ADM1H'
if ($importedLocality->getFeatureCode() == 'ADM1') {
$states[$locality->getAdmin1Code()] = $locality;
// || $importedLocality->getFeatureCode() == 'ADM2H'
} elseif ($importedLocality->getFeatureCode() == 'ADM2') {
$substates[$locality->getAdmin2Code()] = $locality;
}
}
}
if (isset($localityManager) && null != $localityManager) {
$localityManager->flush();
// $localityManager->clear();
}
$progressHelper->finish();
$outputInterface->writeln("Setting state field on cities...");
$log->info("Saving {country} data", ['country' => $country->getName()]);
// Lets update states on city entities. We cannot do it earlier as desired states may not be loaded on city import
if (count($cities)) {
$cityManager = $managerRegistry->getManagerForClass(get_class($cities[0]));
/** @var $city \JJs\Bundle\GeonamesBundle\Entity\City */
$batchSizeCities = 150;
$iCities = 0;
foreach ($cities as $city) {
$iCities++;
if (isset($states[$city->getAdmin1Code()])) {
$city->setState($states[$city->getAdmin1Code()]);
}
if (isset($substates[$city->getAdmin2Code()])) {
$city->setSubState($substates[$city->getAdmin2Code()]);
}
$cityManager->persist($city);
if ($iCities % $batchSizeCities === 0) {
$cityManager->flush();
// $cityManager->clear();
}
}
$cityManager->flush();
// $cityManager->clear();
}
if (count($substates)) {
reset($substates);
$firstKey = key($substates);
$stateManager = $managerRegistry->getManagerForClass(get_class($substates[$firstKey]));
$batchSizeStates = 150;
$iStates = 0;
foreach ($substates as $substate) {
// var_dump($substate->getNameUtf8() . ' (' . $substate->getAdmin1Code() . '::' . $substate->getAdmin2Code() . ')');
$iStates++;
if (isset($states[$substate->getAdmin1Code()])) {
$state = $states[$substate->getAdmin1Code()];
// var_dump('==>' . $state->getNameUtf8());
$substate->setState($state);
$stateManager->persist($substate);
}
if ($iStates % $batchSizeStates === 0) {
$stateManager->flush();
// $stateManager->clear();
}
}
$stateManager->flush();
// $stateManager->clear();
}
// Flush all managers
foreach ($managers as $manager) {
$manager->flush();
$manager->clear();
}
$log->notice("{code} ({country}) data saved", ['code' => $country->getCode(), 'country' => $country->getName()]);
// Close the locality stream
fclose($stream);
}
