/** * Executes the command * * @param InputInterface $input * @param OutputInterface $output */ protected function execute(InputInterface $input, OutputInterface $output) { $output->writeln("Start"); // test to see if threading is available if (!Core_Thread::available()) { die('Threads not supported'); } #var_dump(self::paralel(10, "t1")); // create 2 thread objects $t1 = new Core_Thread(array('xrow\\EzPublishSolrDocsBundle\\Command\\ProcessTestCommand', 'paralel')); $t2 = new Core_Thread(array('xrow\\EzPublishSolrDocsBundle\\Command\\ProcessTestCommand', 'paralel')); // start them $t1->start(10, 't1'); $t2->start(10, 't2'); // keep the program running until the threads finish while ($t1->isAlive() && $t2->isAlive()) { } #- See more at: http://blog.motane.lu/2009/01/02/multithreading-in-php/#sthash.nBeHweEm.dpuf $output->writeln("Ende"); }
function import() { #$this->stopReplication(); $importstartzeit = microtime(true); $from = 0; $till = $this->source->limit(); $this->source->rewind(); if ($this->source->offset() <= $this->source->count()) { $from = $this->source->offset(); } if ($from + $this->source->limit() > $this->source->count()) { $till = $this->source->count(); } else { $till = $from + $this->source->limit(); } $process_to_run = $this->processes_to_run; $i = $from; $next = true; $compcount = $till - $from; #var_dump("total: " . $compcount); #var_dump("possiblesteps: " . round($compcount/$process_to_run) + 1); #die("sdf"); $steps = (int) round($compcount / $process_to_run) + 1; #$till=$till-1; echo "Erster Datensatz: " . $from . "\n"; echo "Letzter Datensatz: " . ($till - 1) . "\n"; echo "Insgesamt: " . $compcount . "\n"; echo "Conc. Processes: " . $process_to_run . "\n"; echo "Steps: " . $steps . "\n"; if ($process_to_run > 1) { if (!Core_Thread::available()) { die('Threads not supported'); } $till = $till - 1; while ($i < $till and $next) { $range = array(); $range["p1"]["from"] = $i; if ($i + $steps - 1 < $till) { $range["p1"]["to"] = $i + $steps - 1; if ($range["p1"]["to"] == $till) { $next = false; } } else { $range["p1"]["to"] = $till; $next = false; } if ($next) { $range["p2"]["from"] = $range["p1"]["to"] + 1; } if ($next and $range["p2"]["from"] + $steps - 1 < $till) { $range["p2"]["to"] = $range["p2"]["from"] + $steps - 1; if ($range["p2"]["to"] == $till) { $next = false; } } else { if ($next) { $range["p2"]["to"] = $till; } $next = false; } if ($process_to_run > 2) { if ($next) { $range["p3"]["from"] = $range["p2"]["to"] + 1; } if ($next and $range["p3"]["from"] + $steps - 1 < $till) { $range["p3"]["to"] = $range["p3"]["from"] + $steps - 1; if ($range["p3"]["to"] == $till) { $next = false; } } else { if ($next) { $range["p3"]["to"] = $till; } $next = false; } } if ($process_to_run > 3) { if ($next) { $range["p4"]["from"] = $range["p3"]["to"] + 1; } if ($next and $range["p4"]["from"] + $steps - 1 < $till) { $range["p4"]["to"] = $range["p4"]["from"] + $steps - 1; if ($range["p4"]["to"] == $till) { $next = false; } } else { if ($next) { $range["p4"]["to"] = $till; } $next = false; } } if ($next) { if ($process_to_run == 4) { $i = $range["p4"]["to"] + 1; } if ($process_to_run == 3) { $i = $range["p3"]["to"] + 1; } if ($process_to_run == 2) { $i = $range["p2"]["to"] + 1; } } $threads = array(); $importforthissession = 0; foreach ($range as $it_name => $iThread) { $threads[$it_name] = new Core_Thread(array('xrow\\EzPublishSolrDocsBundle\\src\\Import\\Process', 'threadedImport')); $threads[$it_name]->start($it_name, $iThread["from"], $iThread["to"], $this); $importforthissession = $importforthissession + ($iThread["to"] + 1 - $iThread["from"]); } $stillalive = true; while ($stillalive) { #sleep(1); $alivecount = 0; foreach ($range as $it_name => $threaddeditem) { if ($threads[$it_name]->isAlive()) { $alivecount++; } } if ($alivecount == 0) { $this->importedint = $this->importedint + $importforthissession; $stillalive = false; } #else sleep(1); // wait until no thread is alive anymore } echo "\rImportiert: " . $this->importedint . " von " . $compcount; } } else { // no Threading for ($i = $from; $i < $till; $i++) { $this->importEntry($this->source->current($i)); $this->importedint = $this->importedint + 1; echo "\rImportiert: " . $this->importedint . " von " . $compcount; } } // Implement threading $durationInMilliseconds = (microtime(true) - $importstartzeit) * 1000; $timing = number_format($durationInMilliseconds, 3, '.', '') . "ms"; if ($durationInMilliseconds > 1000) { $timing = number_format($durationInMilliseconds / 1000, 1, '.', '') . "sec"; } echo "\nDauer Import: " . $timing . "\n"; $average = $compcount / ($durationInMilliseconds / 1000); echo "Average Objects per second: " . number_format($average, 1, '.', '') . "\n"; #$this->startReplication(); }
/** * Executes the command * * @param InputInterface $input * @param OutputInterface $output */ protected function execute(InputInterface $input, OutputInterface $output) { $sourcefile = $input->getOption('source'); $contentTypeIdentifier = $input->getOption('class'); $limitopt = $input->getOption('limit'); $offsetopt = $input->getOption('offset'); try { $startzeit = microtime(true); $output->writeln("Starting Import"); $output->writeln("---------------"); $output->writeln(""); // PREPARING $repository = $this->getContainer()->get('ezpublish.solrapi.repository'); $repository1 = $this->getContainer()->get('ezpublish.solrapi.repository'); $repository2 = $this->getContainer()->get('ezpublish.solrapi.repository'); $contentTypeService = $repository->getContentTypeService(); $locationService = $repository->getLocationService(); $parentLocationId = 2; $location = $locationService->newLocationCreateStruct($parentLocationId); $location1 = $locationService->newLocationCreateStruct($parentLocationId); $location2 = $locationService->newLocationCreateStruct($parentLocationId); $ContentType = $contentTypeService->loadContentTypeByIdentifier($contentTypeIdentifier); $ContentType1 = $contentTypeService->loadContentTypeByIdentifier($contentTypeIdentifier); $ContentType2 = $contentTypeService->loadContentTypeByIdentifier($contentTypeIdentifier); // IMPORTING NOW $offset = 0; $limit = 2000; if ($offsetopt !== null) { $offset = $offsetopt; } if ($limitopt !== null) { $limit = $limitopt; } // test to see if threading is available if (!Core_Thread::available()) { die('Threads not supported'); } #var_dump(self::paralel(10, "t1")); // create 2 thread objects $t1 = new Core_Thread(array('xrow\\EzPublishSolrDocsBundle\\Command\\OdataImportCommand', '_doImport')); $t2 = new Core_Thread(array('xrow\\EzPublishSolrDocsBundle\\Command\\OdataImportCommand', '_doImport')); // start them $t1->start($sourcefile, 0, 10, $location1, $ContentType1, $repository1, $output, $startzeit); $t2->start($sourcefile, 11, 10, $location2, $ContentType2, $repository2, $output, $startzeit); // keep the program running until the threads finish while ($t1->isAlive() && $t2->isAlive()) { } #self::_doImport($sourcefile, $offset, $limit, $location, $ContentType, $repository, $output, $startzeit); /* $source = new OData\Source( $sourcefile, $offset, $limit); $output->writeln("Sourced " . $sourcefile); $output->writeln("---------------"); $output->writeln(""); $import = new Import\Process( $location, $ContentType, $source, $repository ); $output->writeln("Validate and go"); $output->writeln("---------------"); $output->writeln(""); if($import->validate( $source )) { $output->writeln("Import is valid."); $output->writeln("Rows: " . $source->count()); $import->import($source); } else { $output->writeln("Import is NOT valid."); } $output->writeln(""); $output->writeln("---------------"); $output->writeln("Finished Import."); // ECHO TIMING $durationInMilliseconds = (microtime(true) - $startzeit) * 1000; $timing = number_format($durationInMilliseconds, 3, '.', '') . "ms"; if($durationInMilliseconds > 1000) { $timing = number_format($durationInMilliseconds / 1000, 1, '.', '') . "sec"; } $output->writeln("Dauer: " . $timing); */ // DOING COMMIT $client = new \Solarium\Client($this->getContainer()->getParameter('xrow_ez_publish_solr_docs.solrserverconfig')); $update = $client->createUpdate(); $update->addCommit(); $result = $client->update($update); $output->writeln(""); $output->writeln("COMMIT done"); } catch (\eZ\Publish\API\Repository\Exceptions\NotFoundException $e) { $output->writeln($e->getMessage()); } catch (\eZ\Publish\API\Repository\Exceptions\ContentFieldValidationException $e) { $output->writeln($e->getMessage()); } catch (\eZ\Publish\API\Repository\Exceptions\ContentValidationException $e) { $output->writeln($e->getMessage()); } }