/** * Maintain the worker process map and notify the worker of an exited process. * @param bool $block When true, method will block waiting for an exit signal * @return void */ public function reap($block = false) { $map = $this->processes(); while (true) { $pid = pcntl_wait($status, $block === true && $this->daemon->is('parent') ? NULL : WNOHANG); if (!$pid || !isset($map[$pid])) { break; } $alias = $map[$pid]->group; $process = $this->processes[$alias][$pid]; $this->daemon->dispatch(array(Core_Daemon::ON_REAP), array($process, $status)); unset($this->processes[$alias][$pid]); // Keep track of process churn -- failures within a processes min_ttl // If too many failures of new processes occur inside a given interval, that's a problem. // Raise a fatal error to prevent runaway process forking which can be very damaging to a server if ($this->daemon->is('shutdown') || $process->runtime() >= $process->min_ttl) { continue; } foreach ($this->failures as $key => $failure_time) { if ($failure_time + self::CHURN_WINDOW < time()) { unset($this->failures[$key]); } } if (count($this->failures) > self::CHURN_LIMIT) { $this->daemon->fatal_error("Recently forked processes are continuously failing. See error log for additional details."); } } }
/** * Start the engines own loop * * @return void */ public function doWork() { $this->mediator->log('Sunrise is working'); while (!\Core_Daemon::is('shutdown')) { usleep(1000); } }
/** * serializes the build before shutting down * * @return void */ public function tearDown() { if (!\Core_Daemon::is('parent')) { $this->mediator->log('TearDown'); if ($this->build != null) { $this->serializeBuild($this->build); $this->build = null; } } }
/** * Dispatch ON_ERROR event, write an error message to the event log, and restart the worker. * * Part of the Worker API - Use from your worker to log a fatal error message and restart the current process. * * @param $message * @return void */ public function fatal_error($message) { if ($this->daemon->is('parent')) { $this->daemon->fatal_error("Fatal Error: {$message}", $this->alias); } else { $this->daemon->fatal_error("Fatal Error: {$message}\nWorker process will restart", $this->alias); } }
/** * Handle IPC Errors * @param $error * @param int $try Inform error() of repeated failures of the same $error_code * @return boolean Returns true if the operation should be retried. */ public function error($error, $try = 1) { // Create an array of random, moderate size and verify it can be written to shared memory // Return boolean $that = $this; $test = function () use($that) { $arr = array_fill(0, mt_rand(10, 100), mt_rand(1000, 1000 * 1000)); $key = mt_rand(1000 * 1000, 2000 * 1000); @shm_put_var($that->shm, $key, $arr); usleep(5000); return @shm_get_var($that->shm, $key) == $arr; }; switch ($error) { case 0: // Success // Success case 4: // System Interrupt // System Interrupt case MSG_ENOMSG: // No message of desired type // Ignored Errors return true; break; case MSG_EAGAIN: // Temporary Problem, Try Again usleep($this->mediator->backoff(20000, $try)); return true; break; case 13: // Permission Denied $this->mediator->count_error('communication'); $this->mediator->log('Permission Denied: Cannot connect to message queue'); $this->purge_mq(); if (Core_Daemon::is('parent')) { usleep($this->mediator->backoff(100000, $try)); } else { sleep($this->mediator->backoff(3, $try)); } $this->setup_ipc(); return true; break; case 22: // Invalid Argument // Probably because the queue was removed in another process. // Invalid Argument // Probably because the queue was removed in another process. case 43: // Identifier Removed // A message queue was re-created at this address but the resource identifier we have needs to be re-created $this->mediator->count_error('communication'); if (Core_Daemon::is('parent')) { usleep($this->mediator->backoff(20000, $try)); } else { sleep($this->mediator->backoff(2, $try)); } $this->setup_ipc(); return true; break; case self::ERROR_UNKNOWN: // Almost certainly an issue with shared memory $this->mediator->log("Shared Memory I/O Error at Address {$this->mediator->guid}."); $this->mediator->count_error('corruption'); // If this is a worker, all we can do is try to re-attach the shared memory. // Any corruption or OOM errors will be handled by the parent exclusively. if (!Core_Daemon::is('parent')) { sleep($this->mediator->backoff(3, $try)); $this->setup_ipc(); return true; } // If this is the parent, do some diagnostic checks and attempt correction. usleep($this->mediator->backoff(20000, $try)); // Test writing to shared memory using an array that should come to a few kilobytes. for ($i = 0; $i < 2; $i++) { if ($test()) { return true; } // Re-attach the shared memory and try the diagnostic again $this->setup_ipc(); } $this->mediator->log("IPC DIAG: Re-Connect failed to solve the problem."); if (!$this->mediator->daemon->is('parent')) { break; } // Attempt to re-connect the shared memory // See if we can read what's in shared memory and re-write it later $items_to_copy = array(); $items_to_call = array(); for ($i = 0; $i < $this->mediator->call_count; $i++) { $call = @shm_get_var($this->shm, $i); if (!is_object($call)) { continue; } $cached = $this->mediator->get_struct($i); if (!is_object($cached)) { continue; } if ($cached->status == Core_Worker_Mediator::TIMEOUT) { continue; } if ($cached->status == Core_Worker_Mediator::UNCALLED) { $items_to_call[$i] = $call; continue; } $items_to_copy[$i] = $call; } $this->mediator->log("IPC DIAG: Preparing to clean SHM and Reconnect..."); for ($i = 0; $i < 2; $i++) { $this->purge_shm(); $this->setup_ipc(); if (!empty($items_to_copy)) { foreach ($items_to_copy as $key => $value) { @shm_put_var($this->shm, $key, $value); } } if (!$test()) { if (empty($items_to_copy)) { $this->mediator->fatal_error("Shared Memory Failure: Unable to proceed."); } else { $this->mediator->log('IPC DIAG: Purging items from shared memory: ' . implode(', ', array_keys($items_to_copy))); unset($items_to_copy); } } } foreach ($items_to_call as $call) { $this->mediator->retry($call); } return true; default: if ($error) { $this->mediator->log("Message Queue Error {$error}: " . posix_strerror($error)); } if (Core_Daemon::is('parent')) { usleep($this->mediator->backoff(100000, $try)); } else { sleep($this->mediator->backoff(3, $try)); } $this->mediator->count_error('catchall'); $this->setup_ipc(); return false; } }