function peek() { $this->init(); $flags = $this->blocking && !$this->blockingTimeout ? 0 : MSG_IPC_NOWAIT; if (!$this->blockingTimeout || !$this->blocking) { msg_receive($this->seg, 1, $msgtype, $this->maxMsgSize, $message, false, $flags, $errno); } else { $timeout = new Scalr_Util_Timeout($this->blockingTimeout); try { while (!$message && !$timeout->reached()) { if (!msg_receive($this->seg, 1, $msgtype, $this->maxMsgSize, $message, false, $flags, $errno)) { $timeout->sleep(10); } } } catch (Scalr_Util_TimeoutException $e) { return null; } } if ($message) { return unserialize($message); } else { if ($errno == MSG_ENOMSG && !$this->blocking) { return null; } if ($errno == 22) { return null; } throw new Scalr_System_Ipc_Exception($errno ? self::$msgrcv_errors[$errno] : "Cannot receive message", $errno); } }
function enter($data = null) { // who will be first? $iAmFirst = false; try { $this->zookeeper->create($this->path); $iAmFirst = true; } catch (Scalr_Service_Zookeeper_Exception $e) { if ($e->getCode() != Scalr_Service_Zookeeper_Exception::CONFLICT) { throw $e; } } // create child node $this->zookeeper->create("{$this->path}/n", $data ? serialize($data) : null, array(Scalr_Service_Zookeeper::OPT_SEQUENCE => true)); // wait while all nodes will enter barrier try { while ($this->timeout && !$this->timeout->reached() || !$this->timeout) { if ($iAmFirst && $this->capacity() >= $this->quorum || !$iAmFirst && !$this->zookeeper->exists($this->path)) { break; } else { Scalr_Util_Timeout::sleep(100); } } } catch (Exception $e) { // Finally delete barrier znode if ($this->autoDelete) { $this->delete(); } throw $e; } if ($this->autoDelete) { $this->delete(); } }
function acquire() { while (!$this->tryAcquire() && ($this->acquireAttemptTimeout && !$this->acquireAttemptTimeout->reached() || !$this->acquireAttemptTimeout)) { $this->logger->debug(sprintf("Cannot acquire exclusive lock on '%s'. Wait '%s'", $this->path, "100 millis")); Scalr_Util_Timeout::sleep(100); } }
function peek() { $this->init(); do { // Fetch path child nodes if (!$this->children) { //$this->logger->debug("Getting queue path child nodes"); if ($this->blocking) { $start = microtime(true); try { do { $childData = $this->zookeeper->getChildren($this->path); $loop = !$childData->children && ($this->blockingTimeout && !Scalr_Util_Timeout::reached($this->blockingTimeout, $start) || !$this->blockingTimeout); } while ($loop); } catch (Scalr_Util_TimeoutException $e) { return null; } } else { $childData = $this->zookeeper->getChildren($this->path); } if ($childData->children) { sort($childData->children); $this->children = $childData->children; } else { $this->logger->debug("Queue is empty"); if (!$this->blocking) { return null; } } } $this->logger->debug("Children capacity: " . count($this->children)); while ($childName = array_shift($this->children)) { try { $statData = $this->zookeeper->get("{$this->path}/{$childName}"); $this->zookeeper->delete("{$this->path}/{$childName}"); $this->logger->info("Peeked {$this->path}/{$childName} from queue"); return base64_decode($statData->data64); } catch (Exception $wasDeleted) { // Continue loop $this->logger->debug(sprintf("Got error while delete node %s. " . "I think it was processed by another client", "{$this->path}/{$childName}")); } } } while (!$this->children); }
protected function checkMemoryLimit() { if ($this->config["memoryLimit"] && $this->memoryLimitTimeout->reached(false)) { $this->memoryLimitTimeout->reset(); // Check allocated memory $os = Scalr_System_OS::getInstance(); $this->logger->debug("Get process childs"); $gpids = $os->getProcessChilds($this->processPool->getPid()); // Get resident memory size for each process in group $memory = 0; foreach ($gpids as $pid) { $memory += $os->getMemoryUsage($pid, Scalr_System_OS::MEM_RES); } $memory += $os->getMemoryUsage(posix_getpid(), Scalr_System_OS::MEM_RES); $this->logger->debug("Memory usage: {$memory} Kb. Memory limit: {$this->config["memoryLimit"]} Kb"); if ($memory > $this->config["memoryLimit"]) { $this->logger->warn(sprintf("Cronjob allocates %d Kb. Maximum %d Kb is allowed by configuration", $memory, $this->config["memoryLimit"])); // Terminate cronjob posix_kill(posix_getpid(), SIGTERM); return 0; } } return 1; }
protected function forkChild($useBarrier = true) { $this->logger->info("Fork child process"); $pid = pcntl_fork(); if ($pid == -1) { // Cannot fork child throw new Scalr_System_Ipc_Exception("Cannot fork child process"); } else { if ($pid) { // Current process $this->logger->info(sprintf("Child PID: %s was forked", $pid)); $this->childs[$pid] = array('pid' => $pid, 'workStartTime' => false, 'termStartTime' => false); $this->worker->childForked($pid); } else { // Child process try { $this->isChild = true; $this->childPid = posix_getpid(); $this->logger->info("Starting..."); $this->fireChildEvent("start"); if (!$useBarrier) { $this->ready = true; } else { /* $stat = new Scalr_System_Ipc_WorkerStat(); $stat->pid = $this->childPid; $stat->startTime = microtime(true); $stat->ready = true; $this->workersStat[$this->childPid] = $stat; */ // Wait for SIGUSR2 while (!$this->ready) { $this->sleepMillis(10); } } $this->worker->startChild(); if ($this->workQueue) { $memoryTick = new Scalr_Util_Timeout($this->workerMemoryLimitTick); $os = Scalr_System_OS::getInstance(); while ($message = $this->workQueue->peek()) { $t1 = microtime(true); $this->logger->info("Peek message from work queue"); // Notify parent before message handler /* $stat = $this->workersStat[$this->childPid]; $stat->message = $message; $stat->workStartTime = microtime(true); $stat->workEndTime = null; $this->workersStat[$this->childPid] = $stat; */ $this->fireChildEvent("beforeHandleWork", array("microtime" => microtime(true), "message" => $message)); //$this->timeLogger->info("before handle work ($message);; " . (microtime(true) - $t1) . ";;"); //$t1 = microtime(true); $this->worker->handleWork($message); //$this->timeLogger->info("handle work ($message);;; " . (microtime(true) - $t1) . ";"); //$t1 = microtime(true); // Notify parent after message handler $this->fireChildEvent("afterHandleWork", array("message" => $message)); if ($this->workerMemoryLimit && $memoryTick->reached(false)) { $this->fireChildEvent("memoryUsage", array("memory" => $os->getMemoryUsage(posix_getpid(), Scalr_System_OS::MEM_RES))); $memoryTick->reset(); } /* $stat = $this->workersStat[$this->childPid]; $stat->workEndTime = microtime(true); if ($this->workerMemoryLimit && $memoryTick->reached(false)) { $stat->memoryUsage = $os->getMemoryUsage($this->childPid, Scalr_System_OS::MEM_RES); $stat->memoryUsageTime = microtime(true); $memoryTick->reset(); } $this->workersStat[$this->childPid] = $stat; */ //$this->timeLogger->info("after handle work ($message);;;; " . (microtime(true) - $t1)); //$this->logger->info("TIME after handleWork : " . round(microtime(true) - $t1, 4) . " sec"); } } $this->worker->endChild(); $this->logger->info("Done"); } catch (Exception $e) { // Raise fatal error $this->logger->info(sprintf("Unhandled exception in worker process: <%s> '%s'", get_class($e), $e->getMessage())); $this->logger->info(sprintf("Worker process %d terminated (exit code: %d)", $this->childPid, self::$termExitCode)); // Sometimes (in our tests when daemonize=true) parent process doesn't receive SIGCHLD // Sending kill signal will force SIGCHLD // TODO: Consider it deeper posix_kill($this->childPid, SIGKILL); exit(self::$termExitCode); } exit; } } }
private function forkCoordinator() { $this->logger->info("Forking coordinator process"); $pid = pcntl_fork(); if ($pid > 0) { $this->coordinatorPid = $pid; } else { if ($pid == 0) { $this->coordinatorLoop = true; $this->coordinatorPid = posix_getpid(); $ppid = posix_getppid(); $this->nodeRegistry->set(self::REGKEY_COORDINATOR_PROCESS_PID, posix_getpid()); $leaderPath = "{$this->jobZPath}/leader"; $leaderTimeout = new Scalr_Util_Timeout($this->leaderTimeout); $zombyTimeout = new Scalr_Util_Timeout((int) $this->config["tickTime"] * 10); $heartbeatTimeout = new Scalr_Util_Timeout((int) $this->config["tickTime"]); // Track mtime from self node $lastMtime = $this->zookeeper->get("{$this->nodeRegistry->path}/{$this->nodeRegistry->node}")->mtime; while ($this->coordinatorLoop) { $leaderTimeout->reset(); try { $exceptionCounter = 0; while (!$leaderTimeout->reached() && $this->coordinatorLoop) { try { // Terminate myself if parent was killed if (!posix_kill($ppid, 0)) { $this->coordinatorLoop = false; break 2; } // Leader election maybe initiated if ($this->leaderElection->isInitiated()) { $this->logger->info("[coordinator] Someone has initiated leader election"); $this->doLeaderElection(); } // Leader may changed $leaderNodeName = $this->zookeeper->getData($leaderPath); $oldIsLeader = $this->isLeader; $this->isLeader = $leaderNodeName == $this->nodeName; if (!$this->isLeader && $oldIsLeader) { $this->logger->info("[coordinator] I am not longer a leader ('{$this->nodeName}'). " . "Leader is '{$leaderNodeName}'"); } // Check leader znode mtime $leaderStat = $this->zookeeper->get($leaderPath); if ($leaderStat->mtime != $this->leaderMtime) { // Leader had updated it's state $leaderTimeout->reset(); $this->logger->info("[coordinator] Leader is the same"); $this->leaderMtime = $leaderStat->mtime; } if ($this->isLeader) { // Process returned nodes. // Administrator's configured leader may be here if ($c = $this->returnedNodesQueue->capacity()) { $this->logger->info(sprintf("%d node(s) have returned back online", $c)); $votes = array($this->elector->getElectionData()); while ($vote = $this->returnedNodesQueue->peek()) { $votes[] = $vote; } $this->checkElectionResults($votes, false); } // Check zomby nodes if ($zombyTimeout->reached(false)) { $childData = $this->zookeeper->getChildren($this->nodeRegistry->path); foreach ($childData->children as $childName) { $childStat = $this->zookeeper->get("{$this->nodeRegistry->path}/{$childName}"); if ($childStat->mtime < $lastMtime) { // Zomby detected $this->logger->info(sprintf("[coordinator] Cleanup zomby node '%s'", $childName)); $this->zookeeper->deleteRecursive("{$this->nodeRegistry->path}/{$childName}"); } } $zombyTimeout->reset(); $lastMtime = $this->zookeeper->get("{$this->nodeRegistry->path}/{$this->nodeRegistry->node}")->mtime; } } // Node heart beat if ($heartbeatTimeout->reached(false)) { $this->logger->debug(sprintf("[coordinator] '%s' heartbeat", $this->nodeName)); $this->nodeRegistry->touchNode(); $heartbeatTimeout->reset(); } // Poll work queue while ($message = $this->globalWorkQueue->peek()) { $this->logger->info("[coordinator] Put received message into local queue"); $this->processPool->workQueue->put($message); } Scalr_Util_Timeout::sleep(1000); } catch (Exception $e) { $this->logger->error(sprintf("[coordinator] Caught in message loop <%s> %s", get_class($e), $e->getMessage())); if (++$exceptionCounter > $this->coordinatorSlippageLimit) { $this->logger->fatal("[coordinator] Got too many consistent exceptions in main loop. " . "Slippage limit: {$this->coordinatorSlippageLimit} exceed"); posix_kill(posix_getppid(), SIGTERM); exit; } } } } catch (Scalr_Util_TimeoutException $e) { $this->logger->warn("[coordinator] Caught leader timeout exception ({$leaderTimeout->format()})"); $this->logger->info("[coordinator] Start new leader election procedure"); try { $this->leaderElection->initiate($this->nodeRegistry->nodesCapacity()); } catch (Exception $e) { $this->logger->error(sprintf("[coordinator] Caught in leader election <%s> %s", get_class($e), $e->getMessage())); } } } $this->logger->info("[coordinator] Done"); exit; } else { if ($pid == -1) { throw new Scalr_System_Cronjob_Exception("Cannot fork coordinator process"); } } } }