/**
  * Start consumers workers
  */
 public function startConsumersAction()
 {
     $workers = array('feedWorkers' => array('queue' => Queue::QUEUE_FEED_CRAWL, 'callback' => 'crawlFeed'), 'postWorkers' => array('queue' => Queue::QUEUE_POST_PROCEED, 'callback' => 'proceedPost'), 'commentWorkers' => array('queue' => Queue::QUEUE_COMMENTS_CRAWL, 'callback' => 'crawlComments'), 'imageWorkers' => array('queue' => Queue::QUEUE_IMAGE_CRAWL, 'callback' => 'crawlImage'));
     $consumerProcess = new \Importer\ConsumerProcess($this->plugin);
     $importer = new \Importer\Service($this->plugin);
     foreach ($workers as $workerName => $worker) {
         $workersNumber = $this->plugin->config->importer->{$workerName};
         for ($i = 0; $i < $workersNumber; $i++) {
             $pid = $consumerProcess->addConsumer($worker['queue'], array($importer, $worker['callback']));
             if ($pid) {
                 $childs[$pid] = $workerName;
             }
         }
     }
     while (pcntl_waitpid(0, $status) != -1) {
         echo '++';
         foreach ($childs as $pid => $role) {
             $childStatus = pcntl_waitpid($pid, $status, WNOHANG);
             //child died
             if ($childStatus == -1 || $childStatus > 0) {
                 $workerName = $childs[$pid];
                 $worker = $workers[$workerName];
                 unset($childs[$pid]);
                 $pid = $consumerProcess->addConsumer($worker['queue'], array($importer, $worker['callback']));
                 $childs[$pid] = $worker;
                 echo 'restarted worker ', $worker, "\n";
             }
         }
     }
 }
Пример #2
0
 /**
  * start crontab and loop
  */
 public function start()
 {
     $this->logger->info("crontab start");
     $crontab = $this->createCrontab();
     $loop = Factory::create();
     // add periodic timer
     $loop->addPeriodicTimer(1, function () use($crontab) {
         $pid = pcntl_fork();
         if ($pid > 0) {
             return;
         } elseif ($pid == 0) {
             $crontab->start(time());
             exit;
         } else {
             $this->logger->error("could not fork");
             exit;
         }
     });
     // recover the sub processes
     $loop->addPeriodicTimer(60, function () {
         while (($pid = pcntl_waitpid(0, $status, WNOHANG)) > 0) {
             $message = "process exit. pid:" . $pid . ". exit code:" . $status;
             $this->logger->info($message);
         }
     });
     $loop->run();
 }
Пример #3
0
 /**
  * Run a callback in $threadsCount parallel processes
  *
  * @param callable $callback Callback to run
  * @param array[] $argsCollection Array of arguments, two-dimensional
  * @return array (pid => callback result)
  */
 public function run(callable $callback, array $argsCollection)
 {
     $file = tempnam(sys_get_temp_dir(), 'php');
     file_put_contents($file, "<?php\n");
     $children = [];
     foreach ($argsCollection as $key => $args) {
         $pid = pcntl_fork();
         switch ($pid) {
             case -1:
                 throw new RuntimeException(sprintf('Unable to fork thread %d of %d', $key, count($argsCollection)));
             case 0:
                 // child
                 $this->runChild($callback, $args, $file);
                 die(0);
             default:
                 //parent
                 $children[] = $pid;
         }
     }
     foreach ($children as $child) {
         pcntl_waitpid($child, $status);
     }
     $result = [];
     require $file;
     unlink($file);
     return $result;
 }
Пример #4
0
 public static function run()
 {
     if (!function_exists('pcntl_fork')) {
         throw new \Exception('pcntl ext not exists');
     }
     $batchArray = self::getBatchArray();
     $pids = array();
     for ($i = 0; $i < self::$forkCount; $i++) {
         $pids[$i] = pcntl_fork();
         if ($pids[$i] == -1) {
             throw new \Exception('analyzer couldn`t fork');
         } elseif (!$pids[$i]) {
             if (is_array($batchArray[$i]) && count($batchArray[$i]) > 0) {
                 foreach ($batchArray[$i] as $rName => $config) {
                     self::analyzerRun($rName, $config);
                 }
             }
             exit(0);
         }
         if (self::$waitFlag) {
             pcntl_waitpid($pids[$i], $status, WUNTRACED);
             echo "wait {$i} -> " . time() . "\n";
         }
     }
 }
function excute($url = '', $data)
{
    connectRedis();
    $response = array();
    $loop = count($data);
    for ($i = 0; $i < $loop; $i++) {
        # code...
        $pid = pcntl_fork();
        if (!$pid) {
            // sleep(1);
            print "In child {$i}\n";
            if (array_key_exists($i, $data)) {
                $x = 7;
                $k = get_url($data[$i]);
                setRedis("data", $i);
            }
            exit($i);
        }
    }
    #process
    while (pcntl_waitpid(0, $status) != -1) {
        $status = pcntl_wexitstatus($status);
        echo "Child {$status} completed\n";
    }
    // dd($response);
    return $response;
}
Пример #6
0
 /**
  * recover the sub processes
  */
 public function processRecoverCallback()
 {
     while (($pid = pcntl_waitpid(0, $status, WNOHANG)) > 0) {
         $message = "process exit. pid:" . $pid . ". exit code:" . $status;
         $this->logger->info($message);
     }
 }
function pleac_Gathering_Output_from_a_Program()
{
    // Run a command and return its results as a string.
    $output_string = shell_exec('program args');
    // Same as above, using backtick operator.
    $output_string = `program args`;
    // Run a command and return its results as a list of strings,
    // one per line.
    $output_lines = array();
    exec('program args', $output_lines);
    // -----------------------------
    // The only way to execute a program without using the shell is to
    // use pcntl_exec(). However, there is no way to do redirection, so
    // you can't capture its output.
    $pid = pcntl_fork();
    if ($pid == -1) {
        die('cannot fork');
    } elseif ($pid) {
        pcntl_waitpid($pid, $status);
    } else {
        // Note that pcntl_exec() automatically prepends the program name
        // to the array of arguments; the program name cannot be spoofed.
        pcntl_exec($program, array($arg1, $arg2));
    }
}
Пример #8
0
 /**
  * Wait for all child processes to complete
  */
 public function wait()
 {
     // Wait for all children to return
     foreach ($this->child_pid_list as $child_pid) {
         pcntl_waitpid($child_pid, $status);
     }
 }
Пример #9
0
 public function testRunningDaemonWithResistingWorker()
 {
     $writer = $this->getFileWriter();
     $fork = $this->outerManager->fork(function () use($writer) {
         $handler = new NullHandler();
         $builder = new Builder(array('worker1' => function () use($writer) {
             $writer("worker1.call");
         }, 'worker2' => array('startup' => function () use($writer) {
             pcntl_signal(SIGQUIT, SIG_IGN);
             pcntl_signal(SIGINT, SIG_IGN);
             $writer("worker2.startup");
         }, 'loop' => function () use($writer) {
             $writer("worker2.call");
         }, 'interval' => 1)));
         $builder->setLogger(new Logger('test', array($handler)))->setShutdownTimeout(3);
         $daemon = $builder->build();
         $daemon->setProcessName('testing');
         $daemon->run();
     });
     sleep(1);
     $start = time();
     $fork->kill(SIGQUIT);
     while (posix_kill($fork->getPid(), 0)) {
         pcntl_waitpid($fork->getPid(), $status, WNOHANG | WUNTRACED);
         usleep(100000);
     }
     $end = time();
     $diff = $end - $start;
     $this->assertTrue($diff >= 2 && $diff <= 4, 'Has been killed in shutdown interval');
     $content = file_get_contents($this->tempFile);
     $this->assertSame(1, preg_match_all('/worker1\\.call/', $content));
     $this->assertSame(1, preg_match_all('/worker2\\.startup/', $content));
     $calls = preg_match_all('/worker2\\.call/', $content);
     $this->assertTrue($calls >= 3 && $calls <= 5, 'Expected amount of worker2 calls');
 }
Пример #10
0
 public static function receive_sqs_for_multi()
 {
     $t1 = microtime(true);
     $pcount = 3;
     $pstack = array();
     for ($i = 1; $i <= $pcount; $i++) {
         $pid = pcntl_fork();
         if ($pid == -1) {
             die('fork できません');
         } else {
             if ($pid) {
                 // 親プロセスの場合
                 $pstack[$pid] = true;
                 if (count($pstack) >= $pcount) {
                     unset($pstack[pcntl_waitpid(-1, $status, WUNTRACED)]);
                 }
             } else {
                 sleep(1);
                 self::receive_sqs_message();
                 exit;
                 //処理が終わったらexitする。
             }
         }
     }
     //先に処理が進んでしまうので待つ
     while (count($pstack) > 0) {
         unset($pstack[pcntl_waitpid(-1, $status, WUNTRACED)]);
     }
     $t2 = microtime(true);
     $process_time = $t2 - $t1;
     \Cli::write("Process time = " . $process_time);
 }
Пример #11
0
function signal_handler($signo)
{
    echo "child process is ending... signal number is {$signo}\n";
    while (($pid = pcntl_waitpid(-1, $stat, WNOHANG)) > 0) {
        sprintf("child %d terminated.\n", $pid);
    }
}
Пример #12
0
 /**
 * Process in Parallel.
 *
 * Run a function (with no return result) on each item in an array in parallel.
 * Note: This function is only useful if order is not important, and you don't
 * need any return values from the function (i.e. no inter-process communication).
 *
 * @param mixed   $func  A closure function to apply to each item in parallel.
 * @param array   $arr   The array to apply function to.
 * @param integer $procs Number of processes to run in parallel.
 *
 * @return void
 * 
 *
  example to run :
 
  $makeDir = function($a) {
  shell_exec('mkdir '.shellescapearg($a));
  }
 
  // An array to process
  $dirnames = array('a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k');
 
  // Run the process in parallel.
  Multithread::processParallel($makeDir, $dirnames, 8);
 * 
 */
 public static function processParallel($func, array $arr, $procs = NULL)
 {
     // to improve take task 5 by 5 and wait last of the group of 5 !
     if (empty($procs)) {
         $procs = Cpu::getCpuCores();
     }
     // Break array up into $procs chunks.
     $chunks = array_chunk($arr, ceil(count($arr) / $procs));
     $pid = -1;
     $children = array();
     foreach ($chunks as $items) {
         $pid = pcntl_fork();
         if ($pid === -1) {
             die('could not fork');
         } else {
             if ($pid === 0) {
                 // We are the child process. Pass a chunk of items to process.
                 array_walk($items, $func);
                 exit(0);
             } else {
                 // We are the parent.
                 $children[] = $pid;
             }
         }
     }
     // Wait for children to finish.
     foreach ($children as $pid) {
         // We are still the parent.
         pcntl_waitpid($pid, $status);
     }
 }
Пример #13
0
 protected function scryed($total, $workers = 8, array $args)
 {
     $quiet = $this->option('quiet');
     $pids = [];
     for ($i = 0; $i < $workers; $i++) {
         $pids[$i] = pcntl_fork();
         switch ($pids[$i]) {
             case -1:
                 echo "fork error : {$i} \r\n";
                 exit;
             case 0:
                 $limit = floor($total / $workers);
                 $offset = $i * $limit;
                 if ($i == $workers - 1) {
                     $limit = $total - $offset;
                 }
                 $this->info(">>> 一个子进程已开启  |   剩 " . ($workers - $i - 1) . " 个  |  pid = " . getmypid() . " | --limit = {$limit}  |  --offset = {$offset}");
                 sleep(2);
                 // 这个sleep仅为看清上面的info,可删
                 array_push($args, "--limit={$limit}", "--offset={$offset}");
                 $quiet && array_push($args, '--quiet');
                 pcntl_exec('/usr/bin/php', $args, []);
                 // exec("/usr/bin/php5 artisan crawler:model autohome --sych-model=false --offset={$offset} --limit={$limit}");
                 // pcntl_exec 与 exec 同样可以执行命令。区别是exec 不会主动输出到shell控制台中
                 exit;
             default:
                 break;
         }
     }
     foreach ($pids as $pid) {
         $pid && pcntl_waitpid($pid, $status);
     }
 }
Пример #14
0
 public function child_signal_handler($signo, $pid = null, $status = null)
 {
     //If no pid is provided, that means we're getting the signal from the system.  Let's figure out
     //which child process ended
     if (!$pid) {
         $pid = pcntl_waitpid(-1, $status, WNOHANG);
     }
     //Make sure we get all of the exited children
     while ($pid > 0) {
         if ($pid && isset($this->current_jobs[$pid])) {
             $exit_code = pcntl_wexitstatus($status);
             if ($exit_code != 0) {
                 echo "{$pid} exited with status " . $exit_code . "\n";
             }
             unset($this->current_jobs[$pid]);
         } else {
             if ($pid) {
                 //Oh no, our job has finished before this parent process could even note that it had been launched!
                 echo "..... Adding {$pid} to the signal queue ..... \n";
                 $this->signal_queue[$pid] = $status;
             }
         }
         $pid = pcntl_waitpid(-1, $status, WNOHANG);
     }
     return true;
 }
Пример #15
0
 /**
  * run a callback in parrallel thread
  *
  * @param callable $callback
  * @return void
  */
 public function run(\Closure $callback = null)
 {
     $callback = $callback ?: function () {
     };
     $children = array();
     while (!empty($this->works)) {
         $items = array_pop($this->works);
         $pid = pcntl_fork();
         if (-1 == $pid) {
             throw new \RuntimeException('無法使用子程序!');
         } elseif ($pid) {
             //子程序start
             $children[] = $pid;
             $this->onStart($pid);
         } else {
             try {
                 //子程序處理 callback
                 $callback($items);
                 exit(0);
             } catch (\Exception $e) {
                 exit($e->getCode());
             }
         }
     }
     if ($children) {
         foreach ($children as $child) {
             $res = pcntl_waitpid($child, $status);
             if ($res == -1 || $res > 0) {
                 //子程序end
                 $status = pcntl_wexitstatus($status);
                 $this->onExit($child, $status);
             }
         }
     }
 }
Пример #16
0
 public function init()
 {
     $this->setGroupAndUser();
     if ($this->isDaemonActive()) {
         echo 'The Daemon is running already!' . "\n";
         exit;
     }
     $this->startDaemon();
     while (!$this->stopServer) {
         if (count($this->childProcesses, COUNT_RECURSIVE) - count($this->childProcesses) < $this->maxChild) {
             $this->startChildProcess();
         }
         usleep(self::DELAY);
         while ($signaled_pid = pcntl_waitpid(-1, $status, WNOHANG)) {
             if ($signaled_pid == -1) {
                 $this->childProcesses = [];
                 break;
             } else {
                 foreach ($this->childProcesses as $key => $childPids) {
                     if (is_array($childPids)) {
                         foreach ($childPids as $keyPid => $value) {
                             if ($value == $signaled_pid) {
                                 unset($this->childProcesses[$key][$keyPid]);
                             }
                         }
                     }
                 }
             }
         }
     }
 }
Пример #17
0
 private function process()
 {
     $messageType = NULL;
     $messageMaxSize = 1024;
     while (TRUE) {
         if (count($this->childs) < $this->max) {
             echo count($this->childs) . " ";
             if (msg_receive($this->queue, QUEUE_TYPE_START, $messageType, $messageMaxSize, $this->message)) {
                 $pid = pcntl_fork();
                 if ($pid == -1) {
                     die('could not fork' . PHP_EOL);
                 } else {
                     if ($pid) {
                         $this->childs[$pid] = TRUE;
                         $messageType = NULL;
                         $this->message = NULL;
                     } else {
                         sleep(3);
                         $this->complete($messageType, $this->message);
                         exit;
                     }
                 }
                 foreach ($this->childs as $pid => $value) {
                     if (pcntl_waitpid($pid, $status, WNOHANG)) {
                         if (pcntl_wifexited($status)) {
                             unset($this->childs[$pid]);
                         }
                     }
                 }
             }
         }
         sleep(1);
     }
 }
Пример #18
0
 public function testDealerRep()
 {
     $pids[] = $this->forkRepWorker();
     $pids[] = $this->forkRepWorker();
     $loop = new StreamSelectLoop();
     $context = new Context($loop);
     $dealer = $context->getSocket(\ZMQ::SOCKET_DEALER);
     $dealer->bind('ipc://test2.ipc');
     sleep(1);
     $msgs = array();
     $dealer->on('message', function ($msg) use(&$msgs) {
         $msgs[] = $msg;
     });
     $dealer->send(array('A', '', 'foo'));
     $dealer->send(array('B', '', 'bar'));
     $loop->addTimer(1, function () use($loop) {
         $loop->stop();
     });
     $loop->run();
     foreach ($pids as $pid) {
         pcntl_waitpid($pid, $status, WUNTRACED);
     }
     $this->assertCount(2, $msgs);
     $this->assertContains(array('A', '', 'foobar'), $msgs);
     $this->assertContains(array('B', '', 'barbar'), $msgs);
 }
Пример #19
0
 protected function sendShutdown()
 {
     $ret = $this->controller->send("SHUTDOWN");
     foreach ($this->getWorkers() as $worker) {
         pcntl_waitpid($worker->getPid(), $status);
     }
 }
Пример #20
0
 public function waitForThreads()
 {
     foreach ($this->pids as $pid) {
         pcntl_waitpid($pid, $status);
     }
     $this->pids = array();
 }
Пример #21
0
 private function sleepUntilThereIsSomethingInteresting($timeLimit, $child)
 {
     pcntl_signal(SIGALRM, [$this, "alarm"], true);
     pcntl_alarm($timeLimit);
     pcntl_waitpid($child, $status);
     //pcntl_signal_dispatch();
 }
 protected function waitForBackground(InputInterface $input, OutputInterface $output)
 {
     $lastMemory = memory_get_usage(true);
     while (true) {
         $memory = memory_get_usage(true);
         if ($memory != $lastMemory) {
             $output->writeln(sprintf("memory change: %d, from %d to %d", $memory - $lastMemory, $lastMemory, $memory));
         }
         $lastMemory = $memory;
         $status = 0;
         $pid = pcntl_waitpid(-1, $status, WNOHANG);
         if ($pid == 0) {
             // no child process has quit
             //usleep(200 * 1000);
         } else {
             if ($pid > 0) {
                 // child process with pid = $pid exits
                 $exitStatus = pcntl_wexitstatus($status);
                 $this->onChildProcessExit($pid, $exitStatus, $input, $output);
             } else {
                 // error
                 $errno = pcntl_get_last_error();
                 if ($errno == PCNTL_ECHILD) {
                     // all children finished
                     mdebug("No more BackgroundProcessRunner children, continue ...");
                     break;
                 } else {
                     // some other error
                     throw new \RuntimeException("Error waiting for process, error = " . pcntl_strerror($errno));
                 }
             }
         }
     }
     return $this->isFailed ? self::EXIT_CODE_COMMON_ERROR : self::EXIT_CODE_OK;
 }
Пример #23
0
 public final function execute()
 {
     $hasThreads = function_exists('pcntl_signal');
     if (!$hasThreads || Cli::getInstance()->isSimulation()) {
         flush();
         try {
             return $this->executeNoThread();
         } catch (Interrupt $e) {
             throw $e;
         } catch (Exception $e) {
             echo $e;
         }
         return;
     }
     pcntl_signal(SIGCHLD, SIG_IGN);
     $pid = pcntl_fork();
     if ($pid < 1) {
         $this->_run();
         posix_kill(posix_getpid(), 9);
         pcntl_waitpid(posix_getpid(), $temp = 0, WNOHANG);
         pcntl_wifexited($temp);
         exit;
         //Make sure we exit...
     } else {
         $this->pid = $pid;
     }
 }
Пример #24
0
function test_me($desc)
{
    $pipes = null;
    $process = proc_open(__DIR__ . "/test_proc_open.sh", $desc, $pipes);
    $status = proc_get_status($process);
    pcntl_waitpid($status["pid"], $child_status);
}
Пример #25
0
 function test()
 {
     $pipe = stream_socket_pair(STREAM_PF_UNIX, STREAM_SOCK_STREAM, STREAM_IPPROTO_IP);
     $base = new Base();
     $called = 0;
     $base->read($pipe[0], function ($event) use(&$called) {
         $called++;
         fgets($event->fd);
         if ($called === 3) {
             $event->base->halt();
         }
     });
     $pid = pcntl_fork();
     if (!$pid) {
         fwrite($pipe[1], "foo\n");
         usleep(500);
         fwrite($pipe[1], "bar\n");
         usleep(500);
         fwrite($pipe[1], "baz\n");
         usleep(500);
         exit;
     }
     $base->loop();
     pcntl_waitpid($pid, $s);
     $this->assertEquals(3, $called);
 }
Пример #26
0
 public function test_generating_unique_nonce()
 {
     $nonces = [];
     for ($i = 0; $i < 10; ++$i) {
         $nonce = $this->oauth->generateNonce();
         $this->assertFalse(in_array($nonce, $nonces));
         array_push($nonces, $nonce);
     }
     // fork a process to run in parallel
     // to see whether running in a multi-process
     // environment might break it
     $pid = pcntl_fork();
     if ($pid) {
         // parent process runs what is here
         for ($i = 0; $i < 10; ++$i) {
             $nonce = $this->oauth->generateNonce();
             $this->assertFalse(in_array($nonce, $nonces));
             array_push($nonces, $nonce);
         }
         // protect agains Zombie children
         pcntl_waitpid($pid, $status);
     } else {
         // child process runs what is here
         for ($i = 0; $i < 10; ++$i) {
             $nonce = $this->oauth->generateNonce();
             $this->assertFalse(in_array($nonce, $nonces));
             array_push($nonces, $nonce);
         }
         exit(1);
     }
 }
Пример #27
0
 /**
  * 执行同步
  * @return bool
  */
 public function run()
 {
     while (1) {
         $online_user = $this->get_data('Online')->get_online_list();
         $user_num = count($online_user);
         if (empty($online_user)) {
             sleep($this->_set_interval_time);
             #等待一下
             break;
         }
         $threads = array();
         foreach ($this->_sync_tables as $table) {
             $pid = pcntl_fork();
             if ($pid == -1) {
                 echo "FORK ERROR \n";
                 return false;
             } else {
                 if ($pid) {
                     $threads[] = $pid;
                     echo "CREATE THREAD SUCESS CUR THREAD ID IS {$pid}\n";
                 } else {
                     $exec_starttime = microtime(true);
                     $this->sync($online_user, $table);
                     echo "syncdb {$user_num} execute {$table['table_name']} func run " . (microtime(true) - $exec_starttime) . "\n";
                     exit(1);
                 }
             }
         }
         /*
         			foreach($this->_sync_union as $union_table){
                         $pid = pcntl_fork();
                         if($pid == -1){
                             echo "FORK ERROR \n";
                             return false;
                         }else{
                             if($pid){
                                 $threads[] = $pid;
                                 echo "CREATE THREAD SUCESS CUR THREAD ID IS {$pid}\n";
                             }else{
                                 $exec_starttime = microtime(true);
                                 $this->sync_union($online_user,$union_table);
                                 echo "syncdb {$user_num} execute {$union_table['table_name']} func run ".(microtime(true)-$exec_starttime)."\n";
                                 exit(1);
                             }
                         }
                     }*/
         foreach ($threads as $thread_id) {
             pcntl_waitpid($thread_id, $status);
             echo "THREAD {$thread_id} NOW EXIT\n";
             if (pcntl_wifexited($status)) {
                 $c = pcntl_wexitstatus($status);
                 echo "CHILD EXIT PID {$thread_id} STATUS {$c}\n";
             }
         }
         echo "SyncDb SUCESS\n";
         sleep($this->_set_interval_time);
         #等待一下
     }
 }
Пример #28
0
 public function getChildSignal($pid = false, $status = null, $wait = WNOHANG)
 {
     if ($pid) {
         return pcntl_waitpid($pid, $status, $wait);
     } else {
         return pcntl_waitpid(-1, $status, $wait);
     }
 }
Пример #29
0
 /**
  * Protect against zombie process
  * @return void
  * @SuppressWarnings(PHPMD.UnusedLocalVariable)
  */
 private function freeResources()
 {
     foreach ($this->processes as $process) {
         if (pcntl_waitpid($process->getPid(), $status) === -1) {
             throw new \RuntimeException('Error while waiting for process ' . $process->getPid());
         }
     }
 }
Пример #30
0
 function waitall()
 {
     foreach ($this->child_pids as $pid) {
         pcntl_waitpid($pid, $status);
     }
     $this->child_pids = array();
     return $this;
 }