/**
* Maintain the worker process map and notify the worker of an exited process.
* @param bool $block When true, method will block waiting for an exit signal
* @return void
*/
public function reap($block = false)
{
$map = $this->processes();
while (true) {
$pid = pcntl_wait($status, $block === true && $this->daemon->is('parent') ? NULL : WNOHANG);
if (!$pid || !isset($map[$pid])) {
break;
}
$alias = $map[$pid]->group;
$process = $this->processes[$alias][$pid];
$this->daemon->dispatch(array(Core_Daemon::ON_REAP), array($process, $status));
unset($this->processes[$alias][$pid]);
// Keep track of process churn -- failures within a processes min_ttl
// If too many failures of new processes occur inside a given interval, that's a problem.
// Raise a fatal error to prevent runaway process forking which can be very damaging to a server
if ($this->daemon->is('shutdown') || $process->runtime() >= $process->min_ttl) {
continue;
}
foreach ($this->failures as $key => $failure_time) {
if ($failure_time + self::CHURN_WINDOW < time()) {
unset($this->failures[$key]);
}
}
if (count($this->failures) > self::CHURN_LIMIT) {
$this->daemon->fatal_error("Recently forked processes are continuously failing. See error log for additional details.");
}
}
}
/**
* Start the engines own loop
*
* @return void
*/
public function doWork()
{
$this->mediator->log('Sunrise is working');
while (!\Core_Daemon::is('shutdown')) {
usleep(1000);
}
}
/**
* serializes the build before shutting down
*
* @return void
*/
public function tearDown()
{
if (!\Core_Daemon::is('parent')) {
$this->mediator->log('TearDown');
if ($this->build != null) {
$this->serializeBuild($this->build);
$this->build = null;
}
}
}
/**
* Dispatch ON_ERROR event, write an error message to the event log, and restart the worker.
*
* Part of the Worker API - Use from your worker to log a fatal error message and restart the current process.
*
* @param $message
* @return void
*/
public function fatal_error($message)
{
if ($this->daemon->is('parent')) {
$this->daemon->fatal_error("Fatal Error: {$message}", $this->alias);
} else {
$this->daemon->fatal_error("Fatal Error: {$message}\nWorker process will restart", $this->alias);
}
}
/**
* Handle IPC Errors
* @param $error
* @param int $try Inform error() of repeated failures of the same $error_code
* @return boolean Returns true if the operation should be retried.
*/
public function error($error, $try = 1)
{
// Create an array of random, moderate size and verify it can be written to shared memory
// Return boolean
$that = $this;
$test = function () use($that) {
$arr = array_fill(0, mt_rand(10, 100), mt_rand(1000, 1000 * 1000));
$key = mt_rand(1000 * 1000, 2000 * 1000);
@shm_put_var($that->shm, $key, $arr);
usleep(5000);
return @shm_get_var($that->shm, $key) == $arr;
};
switch ($error) {
case 0:
// Success
// Success
case 4:
// System Interrupt
// System Interrupt
case MSG_ENOMSG:
// No message of desired type
// Ignored Errors
return true;
break;
case MSG_EAGAIN:
// Temporary Problem, Try Again
usleep($this->mediator->backoff(20000, $try));
return true;
break;
case 13:
// Permission Denied
$this->mediator->count_error('communication');
$this->mediator->log('Permission Denied: Cannot connect to message queue');
$this->purge_mq();
if (Core_Daemon::is('parent')) {
usleep($this->mediator->backoff(100000, $try));
} else {
sleep($this->mediator->backoff(3, $try));
}
$this->setup_ipc();
return true;
break;
case 22:
// Invalid Argument
// Probably because the queue was removed in another process.
// Invalid Argument
// Probably because the queue was removed in another process.
case 43:
// Identifier Removed
// A message queue was re-created at this address but the resource identifier we have needs to be re-created
$this->mediator->count_error('communication');
if (Core_Daemon::is('parent')) {
usleep($this->mediator->backoff(20000, $try));
} else {
sleep($this->mediator->backoff(2, $try));
}
$this->setup_ipc();
return true;
break;
case self::ERROR_UNKNOWN:
// Almost certainly an issue with shared memory
$this->mediator->log("Shared Memory I/O Error at Address {$this->mediator->guid}.");
$this->mediator->count_error('corruption');
// If this is a worker, all we can do is try to re-attach the shared memory.
// Any corruption or OOM errors will be handled by the parent exclusively.
if (!Core_Daemon::is('parent')) {
sleep($this->mediator->backoff(3, $try));
$this->setup_ipc();
return true;
}
// If this is the parent, do some diagnostic checks and attempt correction.
usleep($this->mediator->backoff(20000, $try));
// Test writing to shared memory using an array that should come to a few kilobytes.
for ($i = 0; $i < 2; $i++) {
if ($test()) {
return true;
}
// Re-attach the shared memory and try the diagnostic again
$this->setup_ipc();
}
$this->mediator->log("IPC DIAG: Re-Connect failed to solve the problem.");
if (!$this->mediator->daemon->is('parent')) {
break;
}
// Attempt to re-connect the shared memory
// See if we can read what's in shared memory and re-write it later
$items_to_copy = array();
$items_to_call = array();
for ($i = 0; $i < $this->mediator->call_count; $i++) {
$call = @shm_get_var($this->shm, $i);
if (!is_object($call)) {
continue;
}
$cached = $this->mediator->get_struct($i);
if (!is_object($cached)) {
continue;
}
if ($cached->status == Core_Worker_Mediator::TIMEOUT) {
continue;
}
if ($cached->status == Core_Worker_Mediator::UNCALLED) {
$items_to_call[$i] = $call;
continue;
}
$items_to_copy[$i] = $call;
}
$this->mediator->log("IPC DIAG: Preparing to clean SHM and Reconnect...");
for ($i = 0; $i < 2; $i++) {
$this->purge_shm();
$this->setup_ipc();
if (!empty($items_to_copy)) {
foreach ($items_to_copy as $key => $value) {
@shm_put_var($this->shm, $key, $value);
}
}
if (!$test()) {
if (empty($items_to_copy)) {
$this->mediator->fatal_error("Shared Memory Failure: Unable to proceed.");
} else {
$this->mediator->log('IPC DIAG: Purging items from shared memory: ' . implode(', ', array_keys($items_to_copy)));
unset($items_to_copy);
}
}
}
foreach ($items_to_call as $call) {
$this->mediator->retry($call);
}
return true;
default:
if ($error) {
$this->mediator->log("Message Queue Error {$error}: " . posix_strerror($error));
}
if (Core_Daemon::is('parent')) {
usleep($this->mediator->backoff(100000, $try));
} else {
sleep($this->mediator->backoff(3, $try));
}
$this->mediator->count_error('catchall');
$this->setup_ipc();
return false;
}
}
