/** * Executes a raw import. * * @param Manager $manager * @param string $filename * @param OutputInterface $output * @param int $bulkSize */ protected function executeRawImport(Manager $manager, $filename, OutputInterface $output, $bulkSize) { $reader = $this->getReader($manager, $filename, false); if (class_exists('\\Symfony\\Component\\Console\\Helper\\ProgressBar')) { $progress = new ProgressBar($output, $reader->count()); $progress->setRedrawFrequency(100); $progress->start(); } else { $progress = new ProgressHelper(); $progress->setRedrawFrequency(100); $progress->start($output, $reader->count()); } foreach ($reader as $key => $document) { $data = $document['_source']; $data['_id'] = $document['_id']; $manager->getConnection()->bulk('index', $document['_type'], $data); if (($key + 1) % $bulkSize == 0) { $manager->commit(); } $progress->advance(); } if (($key + 1) % $bulkSize != 0) { $manager->commit(); } $progress->finish(); $output->writeln(''); }
protected function dumpRowsToAllFiles(Config $config, array $dumpers) { // generate random data and write row by row $this->stopwatch->start('generating_rows', 'generate_dumps'); $this->progress->start($this->output, $config->getFakeNumber()); $unit = floor($config->getFakeNumber() / 100); $this->progress->setRedrawFrequency($unit < 1 ? 1 : $unit); $this->progress->setBarCharacter('<comment>=</comment>'); $this->output->writeln('Generating <info>' . $config->getFakeNumber() . '</info> rows ...'); parent::dumpRowsToAllFiles($config, $dumpers); $this->progress->finish(); $this->output->writeln('Finalizing files ...'); $this->stopwatch->stop('generating_rows'); $this->stopwatch->start('finalizing_files', 'generate_dumps'); }
/** * Exports es index to provided file. * * @param Manager $manager * @param string $filename * @param int $chunkSize * @param OutputInterface $output */ public function exportIndex(Manager $manager, $filename, $chunkSize, OutputInterface $output) { $types = $manager->getTypesMapping(); $repo = $manager->getRepository($types); $results = $this->getResults($repo, $chunkSize); if (class_exists('\\Symfony\\Component\\Console\\Helper\\ProgressBar')) { $progress = new ProgressBar($output, $results->getTotalCount()); $progress->setRedrawFrequency(100); $progress->start(); } else { $progress = new ProgressHelper(); $progress->setRedrawFrequency(100); $progress->start($output, $results->getTotalCount()); } $metadata = ['count' => $results->getTotalCount(), 'date' => date(\DateTime::ISO8601)]; $writer = $this->getWriter($this->getFilePath($filename), $metadata); foreach ($results as $data) { $writer->push(array_intersect_key($data, array_flip(['_id', '_type', '_source']))); $progress->advance(); } $writer->finalize(); $progress->finish(); $output->writeln(''); }
/** * Imports the localities of a specific country into the locality * * @param string|CountryInterface $country Country * @param Filter $filter Locality Filter * @param LoggerInterface $log Import log */ public function importCountry($country, Filter $filter = null, LoggerInterface $log = null, ProgressHelper $progressHelper, OutputInterface $outputInterface) { $log = $log ?: new NullLogger(); $countryRepository = $this->getCountryRepository(); $managerRegistry = $this->getManagerRegistry(); // Load the country if required if (!$country instanceof CountryInterface) { $countryCode = $country; $country = $countryRepository->getCountry($countryCode); if (!$country) { $log->error("Country code '{code}' does not exist in repository {repository}", ['code' => $countryCode, 'repository' => get_class($countryRepository)]); return []; } } // Retrieve the locality TSV stream $path = $this->getLocalityDataPath($country, $log); $log->info("Reading locality data from {path}", ['path' => $path]); $stream = @fopen($path, 'r'); $separator = "\t"; // As a workaround for the lack of support for fgetcsv() without any // enclosure value we can use the backspace character as the enclosure $enclosure = chr(8); // Check that the stream opened successfully, if it failed to open log // an error and return execution to the callee if ($stream === false) { $log->error("Could not read locality stream for {country} ({code})", ['country' => $country->getName(), 'code' => $country->getCode()]); return []; } // Register that we're now importing localities $log->info("Importing {country} localities", ['country' => $country->getName()]); // Iterate over all the localities in the stream $lineNumber = 0; $managers = []; $cities = []; $states = []; $substates = []; $progressHelper->setRedrawFrequency(100); $progressHelper->start($outputInterface, $this->getLinesCount($stream)); $stream = @fopen($path, 'r'); $repositories = []; $batchSize = 150; $i = 0; // while (false !== $row = fgetcsv($stream, 0, $separator, $enclosure)) { $content = file_get_contents($path); $rows = explode("\n", $content); foreach ($rows as $row) { $row = explode($separator, $row); $i++; $progressHelper->advance(); // Increment the line number $lineNumber++; // Log the line number and file $log->debug("Line {line} of {file}", ['line' => $lineNumber, 'file' => $path]); // Skip blank rows if (!$row[0]) { continue; } // Display a warning and skip rows where there are not enough // columns to generate a locality if (count($row) < 17) { $log->warning("Line {line} only has {count} of {expected} columns", ['line' => $lineNumber, 'count' => count($row), 'expected' => 17]); continue; } // User function is called indirectly to allow the createLocality // method to be solely responsible for the generation of locality // information $importedLocality = call_user_func_array([$this, 'createLocality'], $row); // Determine the locality repository for import $featureCode = $importedLocality->getFeatureCode(); // Quick FIX because of french changements /* if ($featureCode == 'ADM1H') { $featureCode = 'ADM1'; } if ($featureCode == 'ADM2H') { $featureCode = 'ADM2'; } */ if (!array_key_exists($featureCode, $repositories)) { $repositories[$featureCode] = $this->getLocalityRepository($featureCode); } $localityRepository = $repositories[$featureCode]; // Skip unsupported geographical features if (!$localityRepository) { $log->debug("{name} skipped ({code} feature not supported)", ['code' => $importedLocality->getFeatureCode(), 'name' => $importedLocality->getNameAscii()]); continue; } // Import the specific locality $locality = $this->importLocality($localityRepository, $importedLocality, $country, $filter, $log); // Skip non-importable localities if (!$locality) { continue; } // Determine the manager for the locality $localityClass = get_class($locality); if (!isset($managers[$localityClass])) { $localityManager = $managerRegistry->getManagerForClass($localityClass); // Ensure the locality manager was loaded if (!$localityManager) { $log->error("No object manager registered for {class}", ['class' => get_class($locality)]); continue; } $managers[$localityClass] = $localityManager; } $localityManager = $managers[$localityClass]; if (null == $localityManager) { continue; } // Persist the locality $localityManager->persist($locality); // $localityManager->clear($locality); if ($i % $batchSize === 0) { $localityManager->flush(); // $localityManager->clear(); } if ($locality instanceof City) { $cities[] = $locality; } elseif ($locality instanceof State) { // || $importedLocality->getFeatureCode() == 'ADM1H' if ($importedLocality->getFeatureCode() == 'ADM1') { $states[$locality->getAdmin1Code()] = $locality; // || $importedLocality->getFeatureCode() == 'ADM2H' } elseif ($importedLocality->getFeatureCode() == 'ADM2') { $substates[$locality->getAdmin2Code()] = $locality; } } } if (isset($localityManager) && null != $localityManager) { $localityManager->flush(); // $localityManager->clear(); } $progressHelper->finish(); $outputInterface->writeln("Setting state field on cities..."); $log->info("Saving {country} data", ['country' => $country->getName()]); // Lets update states on city entities. We cannot do it earlier as desired states may not be loaded on city import if (count($cities)) { $cityManager = $managerRegistry->getManagerForClass(get_class($cities[0])); /** @var $city \JJs\Bundle\GeonamesBundle\Entity\City */ $batchSizeCities = 150; $iCities = 0; foreach ($cities as $city) { $iCities++; if (isset($states[$city->getAdmin1Code()])) { $city->setState($states[$city->getAdmin1Code()]); } if (isset($substates[$city->getAdmin2Code()])) { $city->setSubState($substates[$city->getAdmin2Code()]); } $cityManager->persist($city); if ($iCities % $batchSizeCities === 0) { $cityManager->flush(); // $cityManager->clear(); } } $cityManager->flush(); // $cityManager->clear(); } if (count($substates)) { reset($substates); $firstKey = key($substates); $stateManager = $managerRegistry->getManagerForClass(get_class($substates[$firstKey])); $batchSizeStates = 150; $iStates = 0; foreach ($substates as $substate) { // var_dump($substate->getNameUtf8() . ' (' . $substate->getAdmin1Code() . '::' . $substate->getAdmin2Code() . ')'); $iStates++; if (isset($states[$substate->getAdmin1Code()])) { $state = $states[$substate->getAdmin1Code()]; // var_dump('==>' . $state->getNameUtf8()); $substate->setState($state); $stateManager->persist($substate); } if ($iStates % $batchSizeStates === 0) { $stateManager->flush(); // $stateManager->clear(); } } $stateManager->flush(); // $stateManager->clear(); } // Flush all managers foreach ($managers as $manager) { $manager->flush(); $manager->clear(); } $log->notice("{code} ({country}) data saved", ['code' => $country->getCode(), 'country' => $country->getName()]); // Close the locality stream fclose($stream); }