/**
* Forks, runs code in the children and wait until all finished.
*
* @param int $concurrency The amount of forks.
* @param callable $code The code for the fork.
*/
private function fork($concurrency, callable $code)
{
$manager = new ProcessManager();
for ($i = 0; $i < $concurrency; $i++) {
$manager->fork($code);
}
}
public function testRunningDaemonWithResistingWorker()
{
$writer = $this->getFileWriter();
$fork = $this->outerManager->fork(function () use($writer) {
$handler = new NullHandler();
$builder = new Builder(array('worker1' => function () use($writer) {
$writer("worker1.call");
}, 'worker2' => array('startup' => function () use($writer) {
pcntl_signal(SIGQUIT, SIG_IGN);
pcntl_signal(SIGINT, SIG_IGN);
$writer("worker2.startup");
}, 'loop' => function () use($writer) {
$writer("worker2.call");
}, 'interval' => 1)));
$builder->setLogger(new Logger('test', array($handler)))->setShutdownTimeout(3);
$daemon = $builder->build();
$daemon->setProcessName('testing');
$daemon->run();
});
sleep(1);
$start = time();
$fork->kill(SIGQUIT);
while (posix_kill($fork->getPid(), 0)) {
pcntl_waitpid($fork->getPid(), $status, WNOHANG | WUNTRACED);
usleep(100000);
}
$end = time();
$diff = $end - $start;
$this->assertTrue($diff >= 2 && $diff <= 4, 'Has been killed in shutdown interval');
$content = file_get_contents($this->tempFile);
$this->assertSame(1, preg_match_all('/worker1\\.call/', $content));
$this->assertSame(1, preg_match_all('/worker2\\.startup/', $content));
$calls = preg_match_all('/worker2\\.call/', $content);
$this->assertTrue($calls >= 3 && $calls <= 5, 'Expected amount of worker2 calls');
}
public function execute(InputInterface $input, OutputInterface $output)
{
$configuration = new Configuration(file_get_contents(__DIR__ . '/../../../config/config.json'));
$resolver = new SpawnResolver($configuration, CpuInfo::detect());
$factory = new Factory();
$classname = $resolver->getClassName();
if ($input->getOption('verbose')) {
$outputLogger = new StreamHandler('php://stdout');
} else {
$outputLogger = new NullHandler();
}
$workers = new \ArrayIterator();
for ($i = 1; $i <= $resolver->getSpawnQuantity(); $i++) {
$output->write("Launching Worker <info>{$i}</info> ...");
$logger = new Logger('Worker-' . $i);
$logger->pushHandler($outputLogger);
$logger->pushHandler(new RotatingFileHandler(__DIR__ . '/../../../logs/worker-' . $i . '.logs', 3));
$worker = new Worker('Worker-' . $i, new \GearmanWorker(), $logger);
foreach ($configuration['gearman-servers'] as $server) {
$worker->addServer($server['host'], $server['port']);
}
$worker->setFunction(new $classname($configuration, $logger, $factory));
$workers->append($worker);
$output->writeln("Success !");
}
$manager = new ProcessManager(new EventDispatcher());
$manager->process($workers, function (Worker $worker) {
$worker->run();
});
}
/**
* @param InputInterface $input
* @param OutputInterface $out
* @return int
*/
protected function execute(InputInterface $input, OutputInterface $out)
{
$this->outputHeading($out, "Transfer Patterns");
$scanDate = $this->getNextScanDate();
$nonTimetableConnections = $this->outputTask($out, "Loading non-timetable connections", function () use($scanDate) {
return $this->scheduleProvider->getNonTimetableConnections(strtotime($scanDate . " UTC"));
});
$timetables = $this->outputTask($out, "Loading timetables", function () use($scanDate) {
return $this->scheduleProvider->getTimetableConnections(strtotime("{$scanDate} 00:00 UTC"));
});
$interchange = $this->outputTask($out, "Loading interchange", function () {
return $this->scheduleProvider->getInterchangeTimes();
});
$stations = array_keys($this->stationProvider->getLocations());
$persistence = new TransferPatternPersistence($timetables, $nonTimetableConnections, $interchange);
$this->outputTask($out, "Calculating transfer patterns", function () use($stations, $persistence, $scanDate) {
// $persistence->calculateTransferPatternsForStation(call_user_func($this->dbFactory), "MYB", $scanDate);
// return;
$callable = function ($station) use($persistence, $scanDate) {
$persistence->calculateTransferPatternsForStation(call_user_func($this->dbFactory), $station, $scanDate);
};
$this->processManager->process($stations, $callable, $this->forkStrategy);
$this->processManager->wait();
});
$this->setLastScanDate($scanDate);
$this->outputMemoryUsage($out);
return 0;
}
/**
* @param Job $job
* @return \Spork\Fork
*/
protected function performJob(Job $job)
{
$this->host->disconnect();
return $this->spork->fork(function () use($job) {
$performer = new JobPerformer($this->host, $job, $this->jobCreator, $this->logger);
return $performer->perform();
});
}
/**
* Test large batch sizes
*
* @dataProvider batchProvider
*/
public function testLargeBatchProcessing($rangeEnd)
{
$expected = array_fill(0, $rangeEnd, null);
/** @var Fork $fork */
$fork = $this->manager->process($expected, function ($item) {
return $item;
});
$this->manager->wait();
$this->assertEquals($expected, $fork->getResult());
}
/**
* Tests seeding produces different tokens for each process.
*
* @test
*/
public function testSeedRandom()
{
$mutex = $this->buildRedisMutex(1);
$mutex->seedRandom();
$tokens = [];
$processManager = new ProcessManager();
for ($i = 0; $i < 2; $i++) {
$processManager->fork(function () {
$mutex = $this->buildRedisMutex(1);
$mutex->expects($this->any())->method("evalScript")->willReturn(true);
$token = null;
$mutex->expects($this->any())->method("add")->willReturnCallback(function ($redisAPI, $key, $value, $expire) use(&$token) {
$token = "{$value}";
return true;
});
$mutex->synchronized(function () {
});
return $token;
})->then(function (Fork $fork) use(&$tokens) {
$this->assertArrayNotHasKey($fork->getResult(), $tokens);
$tokens[$fork->getResult()] = $fork->getResult();
});
}
}
/**
* Sends shutdown to child instances, which run the worker(s)
*/
protected function shutdownWorkersFromParent()
{
$this->logger->info("Shutting down all workers");
// send initial signal..
foreach ($this->workers as $worker) {
$name = $worker->getName();
if ($this->countRunning($name)) {
/** @var Fork $fork **/
foreach ($this->forks[$name] as $fork) {
$fork->kill($this->shutdownSignal);
}
}
}
// wait for shutdown
$tries = $this->shutdownTimeout * 10;
for ($try = $tries; $try > 0; $try--) {
BuiltIn::doUsleep(100000);
$resisting = 0;
foreach ($this->workers as $worker) {
$name = $worker->getName();
$resisting += $this->countRunning($name);
}
if (!$resisting) {
break;
} elseif ($try % 10 === 0) {
$this->logger->info("{$resisting} resisting worker(s) still remaining - " . $try / 10 . " seconds till slaughter");
}
}
// slaughter survivors
/** @var Fork[] $forks */
foreach ($this->forks as $name => $forks) {
foreach ($forks as $fork) {
if ($this->reallyRunning($fork)) {
$this->event->dispatch(Event::EVENT_WORKER_KILL, new Event($this->workers[$name]), array($fork));
$fork->kill(SIGKILL);
try {
$fork->wait(true);
} catch (ProcessControlException $e) {
// well, it's a SIG KILL..
}
}
}
}
$this->manager->zombieOkay(true);
}
/**
* Tests that locks will be released automatically.
*
* @param callable $mutexFactory The Mutex factory.
* @test
* @dataProvider provideMutexFactories
*/
public function testLiveness(callable $mutexFactory)
{
$manager = new ProcessManager();
$manager->fork(function () use($mutexFactory) {
$mutex = call_user_func($mutexFactory);
$mutex->synchronized(function () {
exit;
});
});
$manager->wait();
sleep(self::TIMEOUT - 1);
$mutex = call_user_func($mutexFactory);
$mutex->synchronized(function () {
});
}
/**
* Download a file node from requested temporary download URL.
*
* @param array $info
* The file info returned by node_file_info() or public_file_info(),
* with requested temporary download URL.
* @param resource $stream
* Stream resource.
* @param string $key
* The file node key.
*
* @return array
* An array of file information having the following entries:
* - s: File size (bytes).
* - at: An array of file attributes having the following entries:
* - n: File name.
*
* @todo Add range support
* @todo Add integrity check
*/
protected function file_download_url($url, $size, $key, $dest, $try_resume = false)
{
// Open the cipher
$td = mcrypt_module_open(MCRYPT_RIJNDAEL_128, '', 'ctr', '');
// Create key
//$key = MEGAUtil::base64_to_a32($key);
$aeskey = array($key[0] ^ $key[4], $key[1] ^ $key[5], $key[2] ^ $key[6], $key[3] ^ $key[7]);
$aeskey = MEGAUtil::a32_to_str($aeskey);
// Create the IV
$iv = array($key[4], $key[5], 0, 0);
$iv = MEGAUtil::a32_to_str($iv);
// Initialize encryption module for decryption
mcrypt_generic_init($td, $aeskey, $iv);
$tmp_path = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'megagluta' . DIRECTORY_SEPARATOR . md5($aeskey) . DIRECTORY_SEPARATOR;
$next_to_add_file = $tmp_path . 'next_to_add';
$resume_dl = $try_resume && is_dir($tmp_path) && is_file($next_to_add_file);
if (!$resume_dl) {
@mkdir($tmp_path, 0777, true);
}
$chunks = $this->get_chunks_list($size);
$ret = 0;
$id_chunks = 0;
foreach ($chunks as $chunk_start => $chunk_size) {
$process_chunks[] = ['start' => $chunk_start, 'size' => $chunk_size, 'id' => $id_chunks++];
}
ChunkToDownloadIterator::prepare($process_chunks, $tmp_path . 'to_dl' . DIRECTORY_SEPARATOR, $resume_dl);
$manager = new ProcessManager();
$manager->process(null, function ($chunk_info) use($tmp_path, $url) {
$this->chunk_download($url, $tmp_path, $chunk_info['id'], $chunk_info['start'], $chunk_info['size']);
}, new CallbackStrategy(function () {
$batches = [];
for ($i = 0; $i < 10; $i++) {
$batches[] = new ChunkToDownloadIterator($i);
}
return $batches;
}));
$next_chunk_id = 0;
if ($resume_dl) {
$next_chunk_id = intval(file_get_contents($next_to_add_file));
}
$pending_time = 0;
$wait_delay = 1000 * 200;
$time_out = 1000 * 1000 * 10;
while ($next_chunk_id < count($chunks) && ($next_file = $tmp_path . $next_chunk_id . '.chunk')) {
usleep($wait_delay);
$pending_time += $wait_delay;
if ($pending_time > $time_out) {
if (!ChunkToDownloadIterator::add_value_file($next_chunk_id, $process_chunks[$next_chunk_id])) {
$this->chunk_download($url, $tmp_path, $next_chunk_id, $process_chunks[$next_chunk_id]['start'], $process_chunks[$next_chunk_id]['size']);
}
}
if (!is_readable($next_file) || !rename($next_file, $next_file = $next_file . '.reading')) {
continue;
}
$ret += fwrite($dest, mdecrypt_generic($td, file_get_contents($next_file)));
unlink($next_file);
file_put_contents($next_to_add_file, ++$next_chunk_id);
$pending_time = 0;
}
$manager->killAll();
mcrypt_generic_deinit($td);
mcrypt_module_close($td);
return $ret;
}
