function peek()
{
$this->init();
$flags = $this->blocking && !$this->blockingTimeout ? 0 : MSG_IPC_NOWAIT;
if (!$this->blockingTimeout || !$this->blocking) {
msg_receive($this->seg, 1, $msgtype, $this->maxMsgSize, $message, false, $flags, $errno);
} else {
$timeout = new Scalr_Util_Timeout($this->blockingTimeout);
try {
while (!$message && !$timeout->reached()) {
if (!msg_receive($this->seg, 1, $msgtype, $this->maxMsgSize, $message, false, $flags, $errno)) {
$timeout->sleep(10);
}
}
} catch (Scalr_Util_TimeoutException $e) {
return null;
}
}
if ($message) {
return unserialize($message);
} else {
if ($errno == MSG_ENOMSG && !$this->blocking) {
return null;
}
if ($errno == 22) {
return null;
}
throw new Scalr_System_Ipc_Exception($errno ? self::$msgrcv_errors[$errno] : "Cannot receive message", $errno);
}
}
function enter($data = null)
{
// who will be first?
$iAmFirst = false;
try {
$this->zookeeper->create($this->path);
$iAmFirst = true;
} catch (Scalr_Service_Zookeeper_Exception $e) {
if ($e->getCode() != Scalr_Service_Zookeeper_Exception::CONFLICT) {
throw $e;
}
}
// create child node
$this->zookeeper->create("{$this->path}/n", $data ? serialize($data) : null, array(Scalr_Service_Zookeeper::OPT_SEQUENCE => true));
// wait while all nodes will enter barrier
try {
while ($this->timeout && !$this->timeout->reached() || !$this->timeout) {
if ($iAmFirst && $this->capacity() >= $this->quorum || !$iAmFirst && !$this->zookeeper->exists($this->path)) {
break;
} else {
Scalr_Util_Timeout::sleep(100);
}
}
} catch (Exception $e) {
// Finally delete barrier znode
if ($this->autoDelete) {
$this->delete();
}
throw $e;
}
if ($this->autoDelete) {
$this->delete();
}
}
function acquire()
{
while (!$this->tryAcquire() && ($this->acquireAttemptTimeout && !$this->acquireAttemptTimeout->reached() || !$this->acquireAttemptTimeout)) {
$this->logger->debug(sprintf("Cannot acquire exclusive lock on '%s'. Wait '%s'", $this->path, "100 millis"));
Scalr_Util_Timeout::sleep(100);
}
}
function peek()
{
$this->init();
do {
// Fetch path child nodes
if (!$this->children) {
//$this->logger->debug("Getting queue path child nodes");
if ($this->blocking) {
$start = microtime(true);
try {
do {
$childData = $this->zookeeper->getChildren($this->path);
$loop = !$childData->children && ($this->blockingTimeout && !Scalr_Util_Timeout::reached($this->blockingTimeout, $start) || !$this->blockingTimeout);
} while ($loop);
} catch (Scalr_Util_TimeoutException $e) {
return null;
}
} else {
$childData = $this->zookeeper->getChildren($this->path);
}
if ($childData->children) {
sort($childData->children);
$this->children = $childData->children;
} else {
$this->logger->debug("Queue is empty");
if (!$this->blocking) {
return null;
}
}
}
$this->logger->debug("Children capacity: " . count($this->children));
while ($childName = array_shift($this->children)) {
try {
$statData = $this->zookeeper->get("{$this->path}/{$childName}");
$this->zookeeper->delete("{$this->path}/{$childName}");
$this->logger->info("Peeked {$this->path}/{$childName} from queue");
return base64_decode($statData->data64);
} catch (Exception $wasDeleted) {
// Continue loop
$this->logger->debug(sprintf("Got error while delete node %s. " . "I think it was processed by another client", "{$this->path}/{$childName}"));
}
}
} while (!$this->children);
}
protected function checkMemoryLimit()
{
if ($this->config["memoryLimit"] && $this->memoryLimitTimeout->reached(false)) {
$this->memoryLimitTimeout->reset();
// Check allocated memory
$os = Scalr_System_OS::getInstance();
$this->logger->debug("Get process childs");
$gpids = $os->getProcessChilds($this->processPool->getPid());
// Get resident memory size for each process in group
$memory = 0;
foreach ($gpids as $pid) {
$memory += $os->getMemoryUsage($pid, Scalr_System_OS::MEM_RES);
}
$memory += $os->getMemoryUsage(posix_getpid(), Scalr_System_OS::MEM_RES);
$this->logger->debug("Memory usage: {$memory} Kb. Memory limit: {$this->config["memoryLimit"]} Kb");
if ($memory > $this->config["memoryLimit"]) {
$this->logger->warn(sprintf("Cronjob allocates %d Kb. Maximum %d Kb is allowed by configuration", $memory, $this->config["memoryLimit"]));
// Terminate cronjob
posix_kill(posix_getpid(), SIGTERM);
return 0;
}
}
return 1;
}
protected function forkChild($useBarrier = true)
{
$this->logger->info("Fork child process");
$pid = pcntl_fork();
if ($pid == -1) {
// Cannot fork child
throw new Scalr_System_Ipc_Exception("Cannot fork child process");
} else {
if ($pid) {
// Current process
$this->logger->info(sprintf("Child PID: %s was forked", $pid));
$this->childs[$pid] = array('pid' => $pid, 'workStartTime' => false, 'termStartTime' => false);
$this->worker->childForked($pid);
} else {
// Child process
try {
$this->isChild = true;
$this->childPid = posix_getpid();
$this->logger->info("Starting...");
$this->fireChildEvent("start");
if (!$useBarrier) {
$this->ready = true;
} else {
/*
$stat = new Scalr_System_Ipc_WorkerStat();
$stat->pid = $this->childPid;
$stat->startTime = microtime(true);
$stat->ready = true;
$this->workersStat[$this->childPid] = $stat;
*/
// Wait for SIGUSR2
while (!$this->ready) {
$this->sleepMillis(10);
}
}
$this->worker->startChild();
if ($this->workQueue) {
$memoryTick = new Scalr_Util_Timeout($this->workerMemoryLimitTick);
$os = Scalr_System_OS::getInstance();
while ($message = $this->workQueue->peek()) {
$t1 = microtime(true);
$this->logger->info("Peek message from work queue");
// Notify parent before message handler
/*
$stat = $this->workersStat[$this->childPid];
$stat->message = $message;
$stat->workStartTime = microtime(true);
$stat->workEndTime = null;
$this->workersStat[$this->childPid] = $stat;
*/
$this->fireChildEvent("beforeHandleWork", array("microtime" => microtime(true), "message" => $message));
//$this->timeLogger->info("before handle work ($message);; " . (microtime(true) - $t1) . ";;");
//$t1 = microtime(true);
$this->worker->handleWork($message);
//$this->timeLogger->info("handle work ($message);;; " . (microtime(true) - $t1) . ";");
//$t1 = microtime(true);
// Notify parent after message handler
$this->fireChildEvent("afterHandleWork", array("message" => $message));
if ($this->workerMemoryLimit && $memoryTick->reached(false)) {
$this->fireChildEvent("memoryUsage", array("memory" => $os->getMemoryUsage(posix_getpid(), Scalr_System_OS::MEM_RES)));
$memoryTick->reset();
}
/*
$stat = $this->workersStat[$this->childPid];
$stat->workEndTime = microtime(true);
if ($this->workerMemoryLimit && $memoryTick->reached(false)) {
$stat->memoryUsage = $os->getMemoryUsage($this->childPid, Scalr_System_OS::MEM_RES);
$stat->memoryUsageTime = microtime(true);
$memoryTick->reset();
}
$this->workersStat[$this->childPid] = $stat;
*/
//$this->timeLogger->info("after handle work ($message);;;; " . (microtime(true) - $t1));
//$this->logger->info("TIME after handleWork : " . round(microtime(true) - $t1, 4) . " sec");
}
}
$this->worker->endChild();
$this->logger->info("Done");
} catch (Exception $e) {
// Raise fatal error
$this->logger->info(sprintf("Unhandled exception in worker process: <%s> '%s'", get_class($e), $e->getMessage()));
$this->logger->info(sprintf("Worker process %d terminated (exit code: %d)", $this->childPid, self::$termExitCode));
// Sometimes (in our tests when daemonize=true) parent process doesn't receive SIGCHLD
// Sending kill signal will force SIGCHLD
// TODO: Consider it deeper
posix_kill($this->childPid, SIGKILL);
exit(self::$termExitCode);
}
exit;
}
}
}
private function forkCoordinator()
{
$this->logger->info("Forking coordinator process");
$pid = pcntl_fork();
if ($pid > 0) {
$this->coordinatorPid = $pid;
} else {
if ($pid == 0) {
$this->coordinatorLoop = true;
$this->coordinatorPid = posix_getpid();
$ppid = posix_getppid();
$this->nodeRegistry->set(self::REGKEY_COORDINATOR_PROCESS_PID, posix_getpid());
$leaderPath = "{$this->jobZPath}/leader";
$leaderTimeout = new Scalr_Util_Timeout($this->leaderTimeout);
$zombyTimeout = new Scalr_Util_Timeout((int) $this->config["tickTime"] * 10);
$heartbeatTimeout = new Scalr_Util_Timeout((int) $this->config["tickTime"]);
// Track mtime from self node
$lastMtime = $this->zookeeper->get("{$this->nodeRegistry->path}/{$this->nodeRegistry->node}")->mtime;
while ($this->coordinatorLoop) {
$leaderTimeout->reset();
try {
$exceptionCounter = 0;
while (!$leaderTimeout->reached() && $this->coordinatorLoop) {
try {
// Terminate myself if parent was killed
if (!posix_kill($ppid, 0)) {
$this->coordinatorLoop = false;
break 2;
}
// Leader election maybe initiated
if ($this->leaderElection->isInitiated()) {
$this->logger->info("[coordinator] Someone has initiated leader election");
$this->doLeaderElection();
}
// Leader may changed
$leaderNodeName = $this->zookeeper->getData($leaderPath);
$oldIsLeader = $this->isLeader;
$this->isLeader = $leaderNodeName == $this->nodeName;
if (!$this->isLeader && $oldIsLeader) {
$this->logger->info("[coordinator] I am not longer a leader ('{$this->nodeName}'). " . "Leader is '{$leaderNodeName}'");
}
// Check leader znode mtime
$leaderStat = $this->zookeeper->get($leaderPath);
if ($leaderStat->mtime != $this->leaderMtime) {
// Leader had updated it's state
$leaderTimeout->reset();
$this->logger->info("[coordinator] Leader is the same");
$this->leaderMtime = $leaderStat->mtime;
}
if ($this->isLeader) {
// Process returned nodes.
// Administrator's configured leader may be here
if ($c = $this->returnedNodesQueue->capacity()) {
$this->logger->info(sprintf("%d node(s) have returned back online", $c));
$votes = array($this->elector->getElectionData());
while ($vote = $this->returnedNodesQueue->peek()) {
$votes[] = $vote;
}
$this->checkElectionResults($votes, false);
}
// Check zomby nodes
if ($zombyTimeout->reached(false)) {
$childData = $this->zookeeper->getChildren($this->nodeRegistry->path);
foreach ($childData->children as $childName) {
$childStat = $this->zookeeper->get("{$this->nodeRegistry->path}/{$childName}");
if ($childStat->mtime < $lastMtime) {
// Zomby detected
$this->logger->info(sprintf("[coordinator] Cleanup zomby node '%s'", $childName));
$this->zookeeper->deleteRecursive("{$this->nodeRegistry->path}/{$childName}");
}
}
$zombyTimeout->reset();
$lastMtime = $this->zookeeper->get("{$this->nodeRegistry->path}/{$this->nodeRegistry->node}")->mtime;
}
}
// Node heart beat
if ($heartbeatTimeout->reached(false)) {
$this->logger->debug(sprintf("[coordinator] '%s' heartbeat", $this->nodeName));
$this->nodeRegistry->touchNode();
$heartbeatTimeout->reset();
}
// Poll work queue
while ($message = $this->globalWorkQueue->peek()) {
$this->logger->info("[coordinator] Put received message into local queue");
$this->processPool->workQueue->put($message);
}
Scalr_Util_Timeout::sleep(1000);
} catch (Exception $e) {
$this->logger->error(sprintf("[coordinator] Caught in message loop <%s> %s", get_class($e), $e->getMessage()));
if (++$exceptionCounter > $this->coordinatorSlippageLimit) {
$this->logger->fatal("[coordinator] Got too many consistent exceptions in main loop. " . "Slippage limit: {$this->coordinatorSlippageLimit} exceed");
posix_kill(posix_getppid(), SIGTERM);
exit;
}
}
}
} catch (Scalr_Util_TimeoutException $e) {
$this->logger->warn("[coordinator] Caught leader timeout exception ({$leaderTimeout->format()})");
$this->logger->info("[coordinator] Start new leader election procedure");
try {
$this->leaderElection->initiate($this->nodeRegistry->nodesCapacity());
} catch (Exception $e) {
$this->logger->error(sprintf("[coordinator] Caught in leader election <%s> %s", get_class($e), $e->getMessage()));
}
}
}
$this->logger->info("[coordinator] Done");
exit;
} else {
if ($pid == -1) {
throw new Scalr_System_Cronjob_Exception("Cannot fork coordinator process");
}
}
}
}
