| public process ( $data, $callable, Spork\Batch\Strategy\StrategyInterface $strategy = null ) | ||
| $strategy | Spork\Batch\Strategy\StrategyInterface | |
public function execute(InputInterface $input, OutputInterface $output)
{
$configuration = new Configuration(file_get_contents(__DIR__ . '/../../../config/config.json'));
$resolver = new SpawnResolver($configuration, CpuInfo::detect());
$factory = new Factory();
$classname = $resolver->getClassName();
if ($input->getOption('verbose')) {
$outputLogger = new StreamHandler('php://stdout');
} else {
$outputLogger = new NullHandler();
}
$workers = new \ArrayIterator();
for ($i = 1; $i <= $resolver->getSpawnQuantity(); $i++) {
$output->write("Launching Worker <info>{$i}</info> ...");
$logger = new Logger('Worker-' . $i);
$logger->pushHandler($outputLogger);
$logger->pushHandler(new RotatingFileHandler(__DIR__ . '/../../../logs/worker-' . $i . '.logs', 3));
$worker = new Worker('Worker-' . $i, new \GearmanWorker(), $logger);
foreach ($configuration['gearman-servers'] as $server) {
$worker->addServer($server['host'], $server['port']);
}
$worker->setFunction(new $classname($configuration, $logger, $factory));
$workers->append($worker);
$output->writeln("Success !");
}
$manager = new ProcessManager(new EventDispatcher());
$manager->process($workers, function (Worker $worker) {
$worker->run();
});
}
/**
* @param InputInterface $input
* @param OutputInterface $out
* @return int
*/
protected function execute(InputInterface $input, OutputInterface $out)
{
$this->outputHeading($out, "Transfer Patterns");
$scanDate = $this->getNextScanDate();
$nonTimetableConnections = $this->outputTask($out, "Loading non-timetable connections", function () use($scanDate) {
return $this->scheduleProvider->getNonTimetableConnections(strtotime($scanDate . " UTC"));
});
$timetables = $this->outputTask($out, "Loading timetables", function () use($scanDate) {
return $this->scheduleProvider->getTimetableConnections(strtotime("{$scanDate} 00:00 UTC"));
});
$interchange = $this->outputTask($out, "Loading interchange", function () {
return $this->scheduleProvider->getInterchangeTimes();
});
$stations = array_keys($this->stationProvider->getLocations());
$persistence = new TransferPatternPersistence($timetables, $nonTimetableConnections, $interchange);
$this->outputTask($out, "Calculating transfer patterns", function () use($stations, $persistence, $scanDate) {
// $persistence->calculateTransferPatternsForStation(call_user_func($this->dbFactory), "MYB", $scanDate);
// return;
$callable = function ($station) use($persistence, $scanDate) {
$persistence->calculateTransferPatternsForStation(call_user_func($this->dbFactory), $station, $scanDate);
};
$this->processManager->process($stations, $callable, $this->forkStrategy);
$this->processManager->wait();
});
$this->setLastScanDate($scanDate);
$this->outputMemoryUsage($out);
return 0;
}
/**
* Test large batch sizes
*
* @dataProvider batchProvider
*/
public function testLargeBatchProcessing($rangeEnd)
{
$expected = array_fill(0, $rangeEnd, null);
/** @var Fork $fork */
$fork = $this->manager->process($expected, function ($item) {
return $item;
});
$this->manager->wait();
$this->assertEquals($expected, $fork->getResult());
}
/**
* Download a file node from requested temporary download URL.
*
* @param array $info
* The file info returned by node_file_info() or public_file_info(),
* with requested temporary download URL.
* @param resource $stream
* Stream resource.
* @param string $key
* The file node key.
*
* @return array
* An array of file information having the following entries:
* - s: File size (bytes).
* - at: An array of file attributes having the following entries:
* - n: File name.
*
* @todo Add range support
* @todo Add integrity check
*/
protected function file_download_url($url, $size, $key, $dest, $try_resume = false)
{
// Open the cipher
$td = mcrypt_module_open(MCRYPT_RIJNDAEL_128, '', 'ctr', '');
// Create key
//$key = MEGAUtil::base64_to_a32($key);
$aeskey = array($key[0] ^ $key[4], $key[1] ^ $key[5], $key[2] ^ $key[6], $key[3] ^ $key[7]);
$aeskey = MEGAUtil::a32_to_str($aeskey);
// Create the IV
$iv = array($key[4], $key[5], 0, 0);
$iv = MEGAUtil::a32_to_str($iv);
// Initialize encryption module for decryption
mcrypt_generic_init($td, $aeskey, $iv);
$tmp_path = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'megagluta' . DIRECTORY_SEPARATOR . md5($aeskey) . DIRECTORY_SEPARATOR;
$next_to_add_file = $tmp_path . 'next_to_add';
$resume_dl = $try_resume && is_dir($tmp_path) && is_file($next_to_add_file);
if (!$resume_dl) {
@mkdir($tmp_path, 0777, true);
}
$chunks = $this->get_chunks_list($size);
$ret = 0;
$id_chunks = 0;
foreach ($chunks as $chunk_start => $chunk_size) {
$process_chunks[] = ['start' => $chunk_start, 'size' => $chunk_size, 'id' => $id_chunks++];
}
ChunkToDownloadIterator::prepare($process_chunks, $tmp_path . 'to_dl' . DIRECTORY_SEPARATOR, $resume_dl);
$manager = new ProcessManager();
$manager->process(null, function ($chunk_info) use($tmp_path, $url) {
$this->chunk_download($url, $tmp_path, $chunk_info['id'], $chunk_info['start'], $chunk_info['size']);
}, new CallbackStrategy(function () {
$batches = [];
for ($i = 0; $i < 10; $i++) {
$batches[] = new ChunkToDownloadIterator($i);
}
return $batches;
}));
$next_chunk_id = 0;
if ($resume_dl) {
$next_chunk_id = intval(file_get_contents($next_to_add_file));
}
$pending_time = 0;
$wait_delay = 1000 * 200;
$time_out = 1000 * 1000 * 10;
while ($next_chunk_id < count($chunks) && ($next_file = $tmp_path . $next_chunk_id . '.chunk')) {
usleep($wait_delay);
$pending_time += $wait_delay;
if ($pending_time > $time_out) {
if (!ChunkToDownloadIterator::add_value_file($next_chunk_id, $process_chunks[$next_chunk_id])) {
$this->chunk_download($url, $tmp_path, $next_chunk_id, $process_chunks[$next_chunk_id]['start'], $process_chunks[$next_chunk_id]['size']);
}
}
if (!is_readable($next_file) || !rename($next_file, $next_file = $next_file . '.reading')) {
continue;
}
$ret += fwrite($dest, mdecrypt_generic($td, file_get_contents($next_file)));
unlink($next_file);
file_put_contents($next_to_add_file, ++$next_chunk_id);
$pending_time = 0;
}
$manager->killAll();
mcrypt_generic_deinit($td);
mcrypt_module_close($td);
return $ret;
}