process() public méthode

public process ( $data, $callable, Spork\Batch\Strategy\StrategyInterface $strategy = null )
$strategy Spork\Batch\Strategy\StrategyInterface
Exemple #1
0
 public function execute(InputInterface $input, OutputInterface $output)
 {
     $configuration = new Configuration(file_get_contents(__DIR__ . '/../../../config/config.json'));
     $resolver = new SpawnResolver($configuration, CpuInfo::detect());
     $factory = new Factory();
     $classname = $resolver->getClassName();
     if ($input->getOption('verbose')) {
         $outputLogger = new StreamHandler('php://stdout');
     } else {
         $outputLogger = new NullHandler();
     }
     $workers = new \ArrayIterator();
     for ($i = 1; $i <= $resolver->getSpawnQuantity(); $i++) {
         $output->write("Launching Worker <info>{$i}</info> ...");
         $logger = new Logger('Worker-' . $i);
         $logger->pushHandler($outputLogger);
         $logger->pushHandler(new RotatingFileHandler(__DIR__ . '/../../../logs/worker-' . $i . '.logs', 3));
         $worker = new Worker('Worker-' . $i, new \GearmanWorker(), $logger);
         foreach ($configuration['gearman-servers'] as $server) {
             $worker->addServer($server['host'], $server['port']);
         }
         $worker->setFunction(new $classname($configuration, $logger, $factory));
         $workers->append($worker);
         $output->writeln("Success !");
     }
     $manager = new ProcessManager(new EventDispatcher());
     $manager->process($workers, function (Worker $worker) {
         $worker->run();
     });
 }
 /**
  * @param InputInterface $input
  * @param OutputInterface $out
  * @return int
  */
 protected function execute(InputInterface $input, OutputInterface $out)
 {
     $this->outputHeading($out, "Transfer Patterns");
     $scanDate = $this->getNextScanDate();
     $nonTimetableConnections = $this->outputTask($out, "Loading non-timetable connections", function () use($scanDate) {
         return $this->scheduleProvider->getNonTimetableConnections(strtotime($scanDate . " UTC"));
     });
     $timetables = $this->outputTask($out, "Loading timetables", function () use($scanDate) {
         return $this->scheduleProvider->getTimetableConnections(strtotime("{$scanDate} 00:00 UTC"));
     });
     $interchange = $this->outputTask($out, "Loading interchange", function () {
         return $this->scheduleProvider->getInterchangeTimes();
     });
     $stations = array_keys($this->stationProvider->getLocations());
     $persistence = new TransferPatternPersistence($timetables, $nonTimetableConnections, $interchange);
     $this->outputTask($out, "Calculating transfer patterns", function () use($stations, $persistence, $scanDate) {
         //            $persistence->calculateTransferPatternsForStation(call_user_func($this->dbFactory), "MYB", $scanDate);
         //            return;
         $callable = function ($station) use($persistence, $scanDate) {
             $persistence->calculateTransferPatternsForStation(call_user_func($this->dbFactory), $station, $scanDate);
         };
         $this->processManager->process($stations, $callable, $this->forkStrategy);
         $this->processManager->wait();
     });
     $this->setLastScanDate($scanDate);
     $this->outputMemoryUsage($out);
     return 0;
 }
 /**
  * Test large batch sizes
  *
  * @dataProvider batchProvider
  */
 public function testLargeBatchProcessing($rangeEnd)
 {
     $expected = array_fill(0, $rangeEnd, null);
     /** @var Fork $fork */
     $fork = $this->manager->process($expected, function ($item) {
         return $item;
     });
     $this->manager->wait();
     $this->assertEquals($expected, $fork->getResult());
 }
Exemple #4
0
 /**
  * Download a file node from requested temporary download URL.
  *
  * @param array $info
  *   The file info returned by node_file_info() or public_file_info(),
  *   with requested temporary download URL.
  * @param resource $stream
  *   Stream resource.
  * @param string $key
  *   The file node key.
  *
  * @return array
  *   An array of file information having the following entries:
  *   - s: File size (bytes).
  *   - at: An array of file attributes having the following entries:
  *     - n: File name.
  *
  * @todo Add range support
  * @todo Add integrity check
  */
 protected function file_download_url($url, $size, $key, $dest, $try_resume = false)
 {
     // Open the cipher
     $td = mcrypt_module_open(MCRYPT_RIJNDAEL_128, '', 'ctr', '');
     // Create key
     //$key = MEGAUtil::base64_to_a32($key);
     $aeskey = array($key[0] ^ $key[4], $key[1] ^ $key[5], $key[2] ^ $key[6], $key[3] ^ $key[7]);
     $aeskey = MEGAUtil::a32_to_str($aeskey);
     // Create the IV
     $iv = array($key[4], $key[5], 0, 0);
     $iv = MEGAUtil::a32_to_str($iv);
     // Initialize encryption module for decryption
     mcrypt_generic_init($td, $aeskey, $iv);
     $tmp_path = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'megagluta' . DIRECTORY_SEPARATOR . md5($aeskey) . DIRECTORY_SEPARATOR;
     $next_to_add_file = $tmp_path . 'next_to_add';
     $resume_dl = $try_resume && is_dir($tmp_path) && is_file($next_to_add_file);
     if (!$resume_dl) {
         @mkdir($tmp_path, 0777, true);
     }
     $chunks = $this->get_chunks_list($size);
     $ret = 0;
     $id_chunks = 0;
     foreach ($chunks as $chunk_start => $chunk_size) {
         $process_chunks[] = ['start' => $chunk_start, 'size' => $chunk_size, 'id' => $id_chunks++];
     }
     ChunkToDownloadIterator::prepare($process_chunks, $tmp_path . 'to_dl' . DIRECTORY_SEPARATOR, $resume_dl);
     $manager = new ProcessManager();
     $manager->process(null, function ($chunk_info) use($tmp_path, $url) {
         $this->chunk_download($url, $tmp_path, $chunk_info['id'], $chunk_info['start'], $chunk_info['size']);
     }, new CallbackStrategy(function () {
         $batches = [];
         for ($i = 0; $i < 10; $i++) {
             $batches[] = new ChunkToDownloadIterator($i);
         }
         return $batches;
     }));
     $next_chunk_id = 0;
     if ($resume_dl) {
         $next_chunk_id = intval(file_get_contents($next_to_add_file));
     }
     $pending_time = 0;
     $wait_delay = 1000 * 200;
     $time_out = 1000 * 1000 * 10;
     while ($next_chunk_id < count($chunks) && ($next_file = $tmp_path . $next_chunk_id . '.chunk')) {
         usleep($wait_delay);
         $pending_time += $wait_delay;
         if ($pending_time > $time_out) {
             if (!ChunkToDownloadIterator::add_value_file($next_chunk_id, $process_chunks[$next_chunk_id])) {
                 $this->chunk_download($url, $tmp_path, $next_chunk_id, $process_chunks[$next_chunk_id]['start'], $process_chunks[$next_chunk_id]['size']);
             }
         }
         if (!is_readable($next_file) || !rename($next_file, $next_file = $next_file . '.reading')) {
             continue;
         }
         $ret += fwrite($dest, mdecrypt_generic($td, file_get_contents($next_file)));
         unlink($next_file);
         file_put_contents($next_to_add_file, ++$next_chunk_id);
         $pending_time = 0;
     }
     $manager->killAll();
     mcrypt_generic_deinit($td);
     mcrypt_module_close($td);
     return $ret;
 }