public process ( $data, $callable, Spork\Batch\Strategy\StrategyInterface $strategy = null ) | ||
$strategy | Spork\Batch\Strategy\StrategyInterface |
public function execute(InputInterface $input, OutputInterface $output) { $configuration = new Configuration(file_get_contents(__DIR__ . '/../../../config/config.json')); $resolver = new SpawnResolver($configuration, CpuInfo::detect()); $factory = new Factory(); $classname = $resolver->getClassName(); if ($input->getOption('verbose')) { $outputLogger = new StreamHandler('php://stdout'); } else { $outputLogger = new NullHandler(); } $workers = new \ArrayIterator(); for ($i = 1; $i <= $resolver->getSpawnQuantity(); $i++) { $output->write("Launching Worker <info>{$i}</info> ..."); $logger = new Logger('Worker-' . $i); $logger->pushHandler($outputLogger); $logger->pushHandler(new RotatingFileHandler(__DIR__ . '/../../../logs/worker-' . $i . '.logs', 3)); $worker = new Worker('Worker-' . $i, new \GearmanWorker(), $logger); foreach ($configuration['gearman-servers'] as $server) { $worker->addServer($server['host'], $server['port']); } $worker->setFunction(new $classname($configuration, $logger, $factory)); $workers->append($worker); $output->writeln("Success !"); } $manager = new ProcessManager(new EventDispatcher()); $manager->process($workers, function (Worker $worker) { $worker->run(); }); }
/** * @param InputInterface $input * @param OutputInterface $out * @return int */ protected function execute(InputInterface $input, OutputInterface $out) { $this->outputHeading($out, "Transfer Patterns"); $scanDate = $this->getNextScanDate(); $nonTimetableConnections = $this->outputTask($out, "Loading non-timetable connections", function () use($scanDate) { return $this->scheduleProvider->getNonTimetableConnections(strtotime($scanDate . " UTC")); }); $timetables = $this->outputTask($out, "Loading timetables", function () use($scanDate) { return $this->scheduleProvider->getTimetableConnections(strtotime("{$scanDate} 00:00 UTC")); }); $interchange = $this->outputTask($out, "Loading interchange", function () { return $this->scheduleProvider->getInterchangeTimes(); }); $stations = array_keys($this->stationProvider->getLocations()); $persistence = new TransferPatternPersistence($timetables, $nonTimetableConnections, $interchange); $this->outputTask($out, "Calculating transfer patterns", function () use($stations, $persistence, $scanDate) { // $persistence->calculateTransferPatternsForStation(call_user_func($this->dbFactory), "MYB", $scanDate); // return; $callable = function ($station) use($persistence, $scanDate) { $persistence->calculateTransferPatternsForStation(call_user_func($this->dbFactory), $station, $scanDate); }; $this->processManager->process($stations, $callable, $this->forkStrategy); $this->processManager->wait(); }); $this->setLastScanDate($scanDate); $this->outputMemoryUsage($out); return 0; }
/** * Test large batch sizes * * @dataProvider batchProvider */ public function testLargeBatchProcessing($rangeEnd) { $expected = array_fill(0, $rangeEnd, null); /** @var Fork $fork */ $fork = $this->manager->process($expected, function ($item) { return $item; }); $this->manager->wait(); $this->assertEquals($expected, $fork->getResult()); }
/** * Download a file node from requested temporary download URL. * * @param array $info * The file info returned by node_file_info() or public_file_info(), * with requested temporary download URL. * @param resource $stream * Stream resource. * @param string $key * The file node key. * * @return array * An array of file information having the following entries: * - s: File size (bytes). * - at: An array of file attributes having the following entries: * - n: File name. * * @todo Add range support * @todo Add integrity check */ protected function file_download_url($url, $size, $key, $dest, $try_resume = false) { // Open the cipher $td = mcrypt_module_open(MCRYPT_RIJNDAEL_128, '', 'ctr', ''); // Create key //$key = MEGAUtil::base64_to_a32($key); $aeskey = array($key[0] ^ $key[4], $key[1] ^ $key[5], $key[2] ^ $key[6], $key[3] ^ $key[7]); $aeskey = MEGAUtil::a32_to_str($aeskey); // Create the IV $iv = array($key[4], $key[5], 0, 0); $iv = MEGAUtil::a32_to_str($iv); // Initialize encryption module for decryption mcrypt_generic_init($td, $aeskey, $iv); $tmp_path = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'megagluta' . DIRECTORY_SEPARATOR . md5($aeskey) . DIRECTORY_SEPARATOR; $next_to_add_file = $tmp_path . 'next_to_add'; $resume_dl = $try_resume && is_dir($tmp_path) && is_file($next_to_add_file); if (!$resume_dl) { @mkdir($tmp_path, 0777, true); } $chunks = $this->get_chunks_list($size); $ret = 0; $id_chunks = 0; foreach ($chunks as $chunk_start => $chunk_size) { $process_chunks[] = ['start' => $chunk_start, 'size' => $chunk_size, 'id' => $id_chunks++]; } ChunkToDownloadIterator::prepare($process_chunks, $tmp_path . 'to_dl' . DIRECTORY_SEPARATOR, $resume_dl); $manager = new ProcessManager(); $manager->process(null, function ($chunk_info) use($tmp_path, $url) { $this->chunk_download($url, $tmp_path, $chunk_info['id'], $chunk_info['start'], $chunk_info['size']); }, new CallbackStrategy(function () { $batches = []; for ($i = 0; $i < 10; $i++) { $batches[] = new ChunkToDownloadIterator($i); } return $batches; })); $next_chunk_id = 0; if ($resume_dl) { $next_chunk_id = intval(file_get_contents($next_to_add_file)); } $pending_time = 0; $wait_delay = 1000 * 200; $time_out = 1000 * 1000 * 10; while ($next_chunk_id < count($chunks) && ($next_file = $tmp_path . $next_chunk_id . '.chunk')) { usleep($wait_delay); $pending_time += $wait_delay; if ($pending_time > $time_out) { if (!ChunkToDownloadIterator::add_value_file($next_chunk_id, $process_chunks[$next_chunk_id])) { $this->chunk_download($url, $tmp_path, $next_chunk_id, $process_chunks[$next_chunk_id]['start'], $process_chunks[$next_chunk_id]['size']); } } if (!is_readable($next_file) || !rename($next_file, $next_file = $next_file . '.reading')) { continue; } $ret += fwrite($dest, mdecrypt_generic($td, file_get_contents($next_file))); unlink($next_file); file_put_contents($next_to_add_file, ++$next_chunk_id); $pending_time = 0; } $manager->killAll(); mcrypt_generic_deinit($td); mcrypt_module_close($td); return $ret; }