/**
* Start consumers workers
*/
public function startConsumersAction()
{
$workers = array('feedWorkers' => array('queue' => Queue::QUEUE_FEED_CRAWL, 'callback' => 'crawlFeed'), 'postWorkers' => array('queue' => Queue::QUEUE_POST_PROCEED, 'callback' => 'proceedPost'), 'commentWorkers' => array('queue' => Queue::QUEUE_COMMENTS_CRAWL, 'callback' => 'crawlComments'), 'imageWorkers' => array('queue' => Queue::QUEUE_IMAGE_CRAWL, 'callback' => 'crawlImage'));
$consumerProcess = new \Importer\ConsumerProcess($this->plugin);
$importer = new \Importer\Service($this->plugin);
foreach ($workers as $workerName => $worker) {
$workersNumber = $this->plugin->config->importer->{$workerName};
for ($i = 0; $i < $workersNumber; $i++) {
$pid = $consumerProcess->addConsumer($worker['queue'], array($importer, $worker['callback']));
if ($pid) {
$childs[$pid] = $workerName;
}
}
}
while (pcntl_waitpid(0, $status) != -1) {
echo '++';
foreach ($childs as $pid => $role) {
$childStatus = pcntl_waitpid($pid, $status, WNOHANG);
//child died
if ($childStatus == -1 || $childStatus > 0) {
$workerName = $childs[$pid];
$worker = $workers[$workerName];
unset($childs[$pid]);
$pid = $consumerProcess->addConsumer($worker['queue'], array($importer, $worker['callback']));
$childs[$pid] = $worker;
echo 'restarted worker ', $worker, "\n";
}
}
}
}
/**
* start crontab and loop
*/
public function start()
{
$this->logger->info("crontab start");
$crontab = $this->createCrontab();
$loop = Factory::create();
// add periodic timer
$loop->addPeriodicTimer(1, function () use($crontab) {
$pid = pcntl_fork();
if ($pid > 0) {
return;
} elseif ($pid == 0) {
$crontab->start(time());
exit;
} else {
$this->logger->error("could not fork");
exit;
}
});
// recover the sub processes
$loop->addPeriodicTimer(60, function () {
while (($pid = pcntl_waitpid(0, $status, WNOHANG)) > 0) {
$message = "process exit. pid:" . $pid . ". exit code:" . $status;
$this->logger->info($message);
}
});
$loop->run();
}
/**
* Run a callback in $threadsCount parallel processes
*
* @param callable $callback Callback to run
* @param array[] $argsCollection Array of arguments, two-dimensional
* @return array (pid => callback result)
*/
public function run(callable $callback, array $argsCollection)
{
$file = tempnam(sys_get_temp_dir(), 'php');
file_put_contents($file, "<?php\n");
$children = [];
foreach ($argsCollection as $key => $args) {
$pid = pcntl_fork();
switch ($pid) {
case -1:
throw new RuntimeException(sprintf('Unable to fork thread %d of %d', $key, count($argsCollection)));
case 0:
// child
$this->runChild($callback, $args, $file);
die(0);
default:
//parent
$children[] = $pid;
}
}
foreach ($children as $child) {
pcntl_waitpid($child, $status);
}
$result = [];
require $file;
unlink($file);
return $result;
}
public static function run()
{
if (!function_exists('pcntl_fork')) {
throw new \Exception('pcntl ext not exists');
}
$batchArray = self::getBatchArray();
$pids = array();
for ($i = 0; $i < self::$forkCount; $i++) {
$pids[$i] = pcntl_fork();
if ($pids[$i] == -1) {
throw new \Exception('analyzer couldn`t fork');
} elseif (!$pids[$i]) {
if (is_array($batchArray[$i]) && count($batchArray[$i]) > 0) {
foreach ($batchArray[$i] as $rName => $config) {
self::analyzerRun($rName, $config);
}
}
exit(0);
}
if (self::$waitFlag) {
pcntl_waitpid($pids[$i], $status, WUNTRACED);
echo "wait {$i} -> " . time() . "\n";
}
}
}
function excute($url = '', $data)
{
connectRedis();
$response = array();
$loop = count($data);
for ($i = 0; $i < $loop; $i++) {
# code...
$pid = pcntl_fork();
if (!$pid) {
// sleep(1);
print "In child {$i}\n";
if (array_key_exists($i, $data)) {
$x = 7;
$k = get_url($data[$i]);
setRedis("data", $i);
}
exit($i);
}
}
#process
while (pcntl_waitpid(0, $status) != -1) {
$status = pcntl_wexitstatus($status);
echo "Child {$status} completed\n";
}
// dd($response);
return $response;
}
/**
* recover the sub processes
*/
public function processRecoverCallback()
{
while (($pid = pcntl_waitpid(0, $status, WNOHANG)) > 0) {
$message = "process exit. pid:" . $pid . ". exit code:" . $status;
$this->logger->info($message);
}
}
function pleac_Gathering_Output_from_a_Program()
{
// Run a command and return its results as a string.
$output_string = shell_exec('program args');
// Same as above, using backtick operator.
$output_string = `program args`;
// Run a command and return its results as a list of strings,
// one per line.
$output_lines = array();
exec('program args', $output_lines);
// -----------------------------
// The only way to execute a program without using the shell is to
// use pcntl_exec(). However, there is no way to do redirection, so
// you can't capture its output.
$pid = pcntl_fork();
if ($pid == -1) {
die('cannot fork');
} elseif ($pid) {
pcntl_waitpid($pid, $status);
} else {
// Note that pcntl_exec() automatically prepends the program name
// to the array of arguments; the program name cannot be spoofed.
pcntl_exec($program, array($arg1, $arg2));
}
}
/**
* Wait for all child processes to complete
*/
public function wait()
{
// Wait for all children to return
foreach ($this->child_pid_list as $child_pid) {
pcntl_waitpid($child_pid, $status);
}
}
public function testRunningDaemonWithResistingWorker()
{
$writer = $this->getFileWriter();
$fork = $this->outerManager->fork(function () use($writer) {
$handler = new NullHandler();
$builder = new Builder(array('worker1' => function () use($writer) {
$writer("worker1.call");
}, 'worker2' => array('startup' => function () use($writer) {
pcntl_signal(SIGQUIT, SIG_IGN);
pcntl_signal(SIGINT, SIG_IGN);
$writer("worker2.startup");
}, 'loop' => function () use($writer) {
$writer("worker2.call");
}, 'interval' => 1)));
$builder->setLogger(new Logger('test', array($handler)))->setShutdownTimeout(3);
$daemon = $builder->build();
$daemon->setProcessName('testing');
$daemon->run();
});
sleep(1);
$start = time();
$fork->kill(SIGQUIT);
while (posix_kill($fork->getPid(), 0)) {
pcntl_waitpid($fork->getPid(), $status, WNOHANG | WUNTRACED);
usleep(100000);
}
$end = time();
$diff = $end - $start;
$this->assertTrue($diff >= 2 && $diff <= 4, 'Has been killed in shutdown interval');
$content = file_get_contents($this->tempFile);
$this->assertSame(1, preg_match_all('/worker1\\.call/', $content));
$this->assertSame(1, preg_match_all('/worker2\\.startup/', $content));
$calls = preg_match_all('/worker2\\.call/', $content);
$this->assertTrue($calls >= 3 && $calls <= 5, 'Expected amount of worker2 calls');
}
public static function receive_sqs_for_multi()
{
$t1 = microtime(true);
$pcount = 3;
$pstack = array();
for ($i = 1; $i <= $pcount; $i++) {
$pid = pcntl_fork();
if ($pid == -1) {
die('fork できません');
} else {
if ($pid) {
// 親プロセスの場合
$pstack[$pid] = true;
if (count($pstack) >= $pcount) {
unset($pstack[pcntl_waitpid(-1, $status, WUNTRACED)]);
}
} else {
sleep(1);
self::receive_sqs_message();
exit;
//処理が終わったらexitする。
}
}
}
//先に処理が進んでしまうので待つ
while (count($pstack) > 0) {
unset($pstack[pcntl_waitpid(-1, $status, WUNTRACED)]);
}
$t2 = microtime(true);
$process_time = $t2 - $t1;
\Cli::write("Process time = " . $process_time);
}
function signal_handler($signo)
{
echo "child process is ending... signal number is {$signo}\n";
while (($pid = pcntl_waitpid(-1, $stat, WNOHANG)) > 0) {
sprintf("child %d terminated.\n", $pid);
}
}
/**
* Process in Parallel.
*
* Run a function (with no return result) on each item in an array in parallel.
* Note: This function is only useful if order is not important, and you don't
* need any return values from the function (i.e. no inter-process communication).
*
* @param mixed $func A closure function to apply to each item in parallel.
* @param array $arr The array to apply function to.
* @param integer $procs Number of processes to run in parallel.
*
* @return void
*
*
example to run :
$makeDir = function($a) {
shell_exec('mkdir '.shellescapearg($a));
}
// An array to process
$dirnames = array('a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k');
// Run the process in parallel.
Multithread::processParallel($makeDir, $dirnames, 8);
*
*/
public static function processParallel($func, array $arr, $procs = NULL)
{
// to improve take task 5 by 5 and wait last of the group of 5 !
if (empty($procs)) {
$procs = Cpu::getCpuCores();
}
// Break array up into $procs chunks.
$chunks = array_chunk($arr, ceil(count($arr) / $procs));
$pid = -1;
$children = array();
foreach ($chunks as $items) {
$pid = pcntl_fork();
if ($pid === -1) {
die('could not fork');
} else {
if ($pid === 0) {
// We are the child process. Pass a chunk of items to process.
array_walk($items, $func);
exit(0);
} else {
// We are the parent.
$children[] = $pid;
}
}
}
// Wait for children to finish.
foreach ($children as $pid) {
// We are still the parent.
pcntl_waitpid($pid, $status);
}
}
protected function scryed($total, $workers = 8, array $args)
{
$quiet = $this->option('quiet');
$pids = [];
for ($i = 0; $i < $workers; $i++) {
$pids[$i] = pcntl_fork();
switch ($pids[$i]) {
case -1:
echo "fork error : {$i} \r\n";
exit;
case 0:
$limit = floor($total / $workers);
$offset = $i * $limit;
if ($i == $workers - 1) {
$limit = $total - $offset;
}
$this->info(">>> 一个子进程已开启 | 剩 " . ($workers - $i - 1) . " 个 | pid = " . getmypid() . " | --limit = {$limit} | --offset = {$offset}");
sleep(2);
// 这个sleep仅为看清上面的info，可删
array_push($args, "--limit={$limit}", "--offset={$offset}");
$quiet && array_push($args, '--quiet');
pcntl_exec('/usr/bin/php', $args, []);
// exec("/usr/bin/php5 artisan crawler:model autohome --sych-model=false --offset={$offset} --limit={$limit}");
// pcntl_exec 与 exec 同样可以执行命令。区别是exec 不会主动输出到shell控制台中
exit;
default:
break;
}
}
foreach ($pids as $pid) {
$pid && pcntl_waitpid($pid, $status);
}
}
public function child_signal_handler($signo, $pid = null, $status = null)
{
//If no pid is provided, that means we're getting the signal from the system. Let's figure out
//which child process ended
if (!$pid) {
$pid = pcntl_waitpid(-1, $status, WNOHANG);
}
//Make sure we get all of the exited children
while ($pid > 0) {
if ($pid && isset($this->current_jobs[$pid])) {
$exit_code = pcntl_wexitstatus($status);
if ($exit_code != 0) {
echo "{$pid} exited with status " . $exit_code . "\n";
}
unset($this->current_jobs[$pid]);
} else {
if ($pid) {
//Oh no, our job has finished before this parent process could even note that it had been launched!
echo "..... Adding {$pid} to the signal queue ..... \n";
$this->signal_queue[$pid] = $status;
}
}
$pid = pcntl_waitpid(-1, $status, WNOHANG);
}
return true;
}
/**
* run a callback in parrallel thread
*
* @param callable $callback
* @return void
*/
public function run(\Closure $callback = null)
{
$callback = $callback ?: function () {
};
$children = array();
while (!empty($this->works)) {
$items = array_pop($this->works);
$pid = pcntl_fork();
if (-1 == $pid) {
throw new \RuntimeException('無法使用子程序!');
} elseif ($pid) {
//子程序start
$children[] = $pid;
$this->onStart($pid);
} else {
try {
//子程序處理 callback
$callback($items);
exit(0);
} catch (\Exception $e) {
exit($e->getCode());
}
}
}
if ($children) {
foreach ($children as $child) {
$res = pcntl_waitpid($child, $status);
if ($res == -1 || $res > 0) {
//子程序end
$status = pcntl_wexitstatus($status);
$this->onExit($child, $status);
}
}
}
}
public function init()
{
$this->setGroupAndUser();
if ($this->isDaemonActive()) {
echo 'The Daemon is running already!' . "\n";
exit;
}
$this->startDaemon();
while (!$this->stopServer) {
if (count($this->childProcesses, COUNT_RECURSIVE) - count($this->childProcesses) < $this->maxChild) {
$this->startChildProcess();
}
usleep(self::DELAY);
while ($signaled_pid = pcntl_waitpid(-1, $status, WNOHANG)) {
if ($signaled_pid == -1) {
$this->childProcesses = [];
break;
} else {
foreach ($this->childProcesses as $key => $childPids) {
if (is_array($childPids)) {
foreach ($childPids as $keyPid => $value) {
if ($value == $signaled_pid) {
unset($this->childProcesses[$key][$keyPid]);
}
}
}
}
}
}
}
}
private function process()
{
$messageType = NULL;
$messageMaxSize = 1024;
while (TRUE) {
if (count($this->childs) < $this->max) {
echo count($this->childs) . " ";
if (msg_receive($this->queue, QUEUE_TYPE_START, $messageType, $messageMaxSize, $this->message)) {
$pid = pcntl_fork();
if ($pid == -1) {
die('could not fork' . PHP_EOL);
} else {
if ($pid) {
$this->childs[$pid] = TRUE;
$messageType = NULL;
$this->message = NULL;
} else {
sleep(3);
$this->complete($messageType, $this->message);
exit;
}
}
foreach ($this->childs as $pid => $value) {
if (pcntl_waitpid($pid, $status, WNOHANG)) {
if (pcntl_wifexited($status)) {
unset($this->childs[$pid]);
}
}
}
}
}
sleep(1);
}
}
public function testDealerRep()
{
$pids[] = $this->forkRepWorker();
$pids[] = $this->forkRepWorker();
$loop = new StreamSelectLoop();
$context = new Context($loop);
$dealer = $context->getSocket(\ZMQ::SOCKET_DEALER);
$dealer->bind('ipc://test2.ipc');
sleep(1);
$msgs = array();
$dealer->on('message', function ($msg) use(&$msgs) {
$msgs[] = $msg;
});
$dealer->send(array('A', '', 'foo'));
$dealer->send(array('B', '', 'bar'));
$loop->addTimer(1, function () use($loop) {
$loop->stop();
});
$loop->run();
foreach ($pids as $pid) {
pcntl_waitpid($pid, $status, WUNTRACED);
}
$this->assertCount(2, $msgs);
$this->assertContains(array('A', '', 'foobar'), $msgs);
$this->assertContains(array('B', '', 'barbar'), $msgs);
}
protected function sendShutdown()
{
$ret = $this->controller->send("SHUTDOWN");
foreach ($this->getWorkers() as $worker) {
pcntl_waitpid($worker->getPid(), $status);
}
}
public function waitForThreads()
{
foreach ($this->pids as $pid) {
pcntl_waitpid($pid, $status);
}
$this->pids = array();
}
private function sleepUntilThereIsSomethingInteresting($timeLimit, $child)
{
pcntl_signal(SIGALRM, [$this, "alarm"], true);
pcntl_alarm($timeLimit);
pcntl_waitpid($child, $status);
//pcntl_signal_dispatch();
}
protected function waitForBackground(InputInterface $input, OutputInterface $output)
{
$lastMemory = memory_get_usage(true);
while (true) {
$memory = memory_get_usage(true);
if ($memory != $lastMemory) {
$output->writeln(sprintf("memory change: %d, from %d to %d", $memory - $lastMemory, $lastMemory, $memory));
}
$lastMemory = $memory;
$status = 0;
$pid = pcntl_waitpid(-1, $status, WNOHANG);
if ($pid == 0) {
// no child process has quit
//usleep(200 * 1000);
} else {
if ($pid > 0) {
// child process with pid = $pid exits
$exitStatus = pcntl_wexitstatus($status);
$this->onChildProcessExit($pid, $exitStatus, $input, $output);
} else {
// error
$errno = pcntl_get_last_error();
if ($errno == PCNTL_ECHILD) {
// all children finished
mdebug("No more BackgroundProcessRunner children, continue ...");
break;
} else {
// some other error
throw new \RuntimeException("Error waiting for process, error = " . pcntl_strerror($errno));
}
}
}
}
return $this->isFailed ? self::EXIT_CODE_COMMON_ERROR : self::EXIT_CODE_OK;
}
public final function execute()
{
$hasThreads = function_exists('pcntl_signal');
if (!$hasThreads || Cli::getInstance()->isSimulation()) {
flush();
try {
return $this->executeNoThread();
} catch (Interrupt $e) {
throw $e;
} catch (Exception $e) {
echo $e;
}
return;
}
pcntl_signal(SIGCHLD, SIG_IGN);
$pid = pcntl_fork();
if ($pid < 1) {
$this->_run();
posix_kill(posix_getpid(), 9);
pcntl_waitpid(posix_getpid(), $temp = 0, WNOHANG);
pcntl_wifexited($temp);
exit;
//Make sure we exit...
} else {
$this->pid = $pid;
}
}
function test_me($desc)
{
$pipes = null;
$process = proc_open(__DIR__ . "/test_proc_open.sh", $desc, $pipes);
$status = proc_get_status($process);
pcntl_waitpid($status["pid"], $child_status);
}
function test()
{
$pipe = stream_socket_pair(STREAM_PF_UNIX, STREAM_SOCK_STREAM, STREAM_IPPROTO_IP);
$base = new Base();
$called = 0;
$base->read($pipe[0], function ($event) use(&$called) {
$called++;
fgets($event->fd);
if ($called === 3) {
$event->base->halt();
}
});
$pid = pcntl_fork();
if (!$pid) {
fwrite($pipe[1], "foo\n");
usleep(500);
fwrite($pipe[1], "bar\n");
usleep(500);
fwrite($pipe[1], "baz\n");
usleep(500);
exit;
}
$base->loop();
pcntl_waitpid($pid, $s);
$this->assertEquals(3, $called);
}
public function test_generating_unique_nonce()
{
$nonces = [];
for ($i = 0; $i < 10; ++$i) {
$nonce = $this->oauth->generateNonce();
$this->assertFalse(in_array($nonce, $nonces));
array_push($nonces, $nonce);
}
// fork a process to run in parallel
// to see whether running in a multi-process
// environment might break it
$pid = pcntl_fork();
if ($pid) {
// parent process runs what is here
for ($i = 0; $i < 10; ++$i) {
$nonce = $this->oauth->generateNonce();
$this->assertFalse(in_array($nonce, $nonces));
array_push($nonces, $nonce);
}
// protect agains Zombie children
pcntl_waitpid($pid, $status);
} else {
// child process runs what is here
for ($i = 0; $i < 10; ++$i) {
$nonce = $this->oauth->generateNonce();
$this->assertFalse(in_array($nonce, $nonces));
array_push($nonces, $nonce);
}
exit(1);
}
}
/**
* 执行同步
* @return bool
*/
public function run()
{
while (1) {
$online_user = $this->get_data('Online')->get_online_list();
$user_num = count($online_user);
if (empty($online_user)) {
sleep($this->_set_interval_time);
#等待一下
break;
}
$threads = array();
foreach ($this->_sync_tables as $table) {
$pid = pcntl_fork();
if ($pid == -1) {
echo "FORK ERROR \n";
return false;
} else {
if ($pid) {
$threads[] = $pid;
echo "CREATE THREAD SUCESS CUR THREAD ID IS {$pid}\n";
} else {
$exec_starttime = microtime(true);
$this->sync($online_user, $table);
echo "syncdb {$user_num} execute {$table['table_name']} func run " . (microtime(true) - $exec_starttime) . "\n";
exit(1);
}
}
}
/*
foreach($this->_sync_union as $union_table){
$pid = pcntl_fork();
if($pid == -1){
echo "FORK ERROR \n";
return false;
}else{
if($pid){
$threads[] = $pid;
echo "CREATE THREAD SUCESS CUR THREAD ID IS {$pid}\n";
}else{
$exec_starttime = microtime(true);
$this->sync_union($online_user,$union_table);
echo "syncdb {$user_num} execute {$union_table['table_name']} func run ".(microtime(true)-$exec_starttime)."\n";
exit(1);
}
}
}*/
foreach ($threads as $thread_id) {
pcntl_waitpid($thread_id, $status);
echo "THREAD {$thread_id} NOW EXIT\n";
if (pcntl_wifexited($status)) {
$c = pcntl_wexitstatus($status);
echo "CHILD EXIT PID {$thread_id} STATUS {$c}\n";
}
}
echo "SyncDb SUCESS\n";
sleep($this->_set_interval_time);
#等待一下
}
}
public function getChildSignal($pid = false, $status = null, $wait = WNOHANG)
{
if ($pid) {
return pcntl_waitpid($pid, $status, $wait);
} else {
return pcntl_waitpid(-1, $status, $wait);
}
}
/**
* Protect against zombie process
* @return void
* @SuppressWarnings(PHPMD.UnusedLocalVariable)
*/
private function freeResources()
{
foreach ($this->processes as $process) {
if (pcntl_waitpid($process->getPid(), $status) === -1) {
throw new \RuntimeException('Error while waiting for process ' . $process->getPid());
}
}
}
function waitall()
{
foreach ($this->child_pids as $pid) {
pcntl_waitpid($pid, $status);
}
$this->child_pids = array();
return $this;
}
