protected function onPoll($events, $read, $write)
{
if ($events) {
foreach ($read as $socket) {
if ($socket === $this->socket) {
$zmsg = new Zmsg($this->socket);
$zmsg->recv();
if ($this->verbose) {
echo "I: received message:", PHP_EOL, $zmsg->__toString(), PHP_EOL;
}
$sender = $zmsg->pop();
$zmsg->pop();
//empty
$header = $zmsg->pop();
if ($header == Commands::W_WORKER) {
$this->process($sender, $zmsg);
} else {
echo "E: invalid header `{$header}` in message", PHP_EOL, $zmsg->__toString(), PHP_EOL, PHP_EOL;
}
}
}
}
$this->generateTasks();
$this->sendHeartbeats();
}
function broker_task()
{
// Prepare our context and sockets
$context = new ZMQContext();
$frontend = new ZMQSocket($context, ZMQ::SOCKET_ROUTER);
$backend = new ZMQSocket($context, ZMQ::SOCKET_ROUTER);
$frontend->bind("tcp://*:5555");
$backend->bind("tcp://*:5556");
// Initialize poll set
$poll = new ZMQPoll();
$poll->add($frontend, ZMQ::POLL_IN);
$poll->add($backend, ZMQ::POLL_IN);
$read = $write = array();
while (true) {
$events = $poll->poll($read, $write);
foreach ($read as $socket) {
$zmsg = new Zmsg($socket);
$zmsg->recv();
if ($socket === $frontend) {
$zmsg->push("W");
$zmsg->set_socket($backend)->send();
} elseif ($socket === $backend) {
$zmsg->pop();
$zmsg->push("C");
$zmsg->set_socket($frontend)->send();
}
}
}
}
function worker_thread($self)
{
$context = new ZMQContext();
$worker = $context->getSocket(ZMQ::SOCKET_REQ);
$endpoint = sprintf("ipc://%s-localbe.ipc", $self);
$worker->connect($endpoint);
// Tell broker we're ready for work
$worker->send("READY");
while (true) {
$zmsg = new Zmsg($worker);
$zmsg->recv();
sleep(mt_rand(0, 2));
$zmsg->send();
}
}
/**
* This is the main listen and process loop
*/
public function listen()
{
$read = $write = array();
// Get and process messages forever or until interrupted
while (true) {
$poll = new ZMQPoll();
$poll->add($this->socket, ZMQ::POLL_IN);
$events = $poll->poll($read, $write, HEARTBEAT_INTERVAL);
// Process next input message, if any
if ($events) {
$zmsg = new Zmsg($this->socket);
$zmsg->recv();
if ($this->verbose) {
echo "I: received message:", PHP_EOL, $zmsg->__toString();
}
$sender = $zmsg->pop();
$empty = $zmsg->pop();
$header = $zmsg->pop();
if ($header == MDPC_CLIENT) {
$this->client_process($sender, $zmsg);
} else {
if ($header == MDPW_WORKER) {
$this->worker_process($sender, $zmsg);
} else {
echo "E: invalid message", PHP_EOL, $zmsg->__toString();
}
}
}
// Disconnect and delete any expired workers
// Send heartbeats to idle workers if needed
if (microtime(true) > $this->heartbeat_at) {
$this->purge_workers();
foreach ($this->workers as $worker) {
$this->worker_send($worker, MDPW_HEARTBEAT, NULL, NULL);
}
$this->heartbeat_at = microtime(true) + HEARTBEAT_INTERVAL / 1000;
}
}
}
protected function onPoll($events, $read, $write)
{
parent::onPoll($events, $read, $write);
if ($events > 0) {
foreach ($read as $socket) {
//handle publisher
if ($socket === $this->frontedSocket) {
$zmsg = new Zmsg($this->frontedSocket);
$zmsg->recv();
if ($this->verbose) {
echo "I: received message from publisher size: ";
echo strlen($zmsg->__toString()), PHP_EOL;
}
$zmsg->unwrap();
//time
if ($this->queueLimit > sizeof($this->queue)) {
array_unshift($this->queue, $zmsg->pop());
}
}
}
}
}
/**
* Start listen for messages in loop
* @throws \Exception
*/
public function listen()
{
if (!$this->listener) {
throw new \Exception("Empty listener");
}
$this->isListen = true;
while ($this->isListen) {
$zmsg = new Zmsg($this->socket);
$zmsg->recv();
if ($this->verbose) {
echo "I: received message from broker:", PHP_EOL;
echo $zmsg->__toString(), PHP_EOL;
}
$time = $zmsg->unwrap();
if (!$this->normalDelay) {
$this->normalDelay = microtime(true) * 1000 - (double) $time;
}
$delayTime = microtime(true) * 1000 - (double) $time;
if ($this->misser && $delayTime > $this->normalDelay + $this->maxAllowedDelay) {
call_user_func($this->misser, $zmsg->pop(), $time, $delayTime);
}
call_user_func($this->listener, $zmsg->pop(), $time);
}
}
// For workers
// Queue of available workers
$available_workers = 0;
$worker_queue = array();
$read = $write = array();
while (true) {
$poll = new ZMQPoll();
$poll->add($backend, ZMQ::POLL_IN);
// Poll frontend only if we have available workers
if ($available_workers) {
$poll->add($frontend, ZMQ::POLL_IN);
}
$events = $poll->poll($read, $write);
foreach ($read as $socket) {
$zmsg = new Zmsg($socket);
$zmsg->recv();
// Handle worker activity on backend
if ($socket === $backend) {
// Use worker address for LRU routing
assert($available_workers < MAX_WORKERS);
array_push($worker_queue, $zmsg->unwrap());
$available_workers++;
// Return reply to client if it's not a READY
if ($zmsg->address() != "READY") {
$zmsg->set_socket($frontend)->send();
}
} elseif ($socket === $frontend) {
// Now get next client request, route to next worker
// REQ socket in worker needs an envelope delimiter
// Dequeue and drop the next worker address
$zmsg->wrap(array_shift($worker_queue), "");
/**
* Run a self test of the Zmsg class.
*
* @return boolean
* @todo See if assert returns
*/
public static function test()
{
$result = true;
$context = new ZMQContext();
$output = new ZMQSocket($context, ZMQ::SOCKET_XREQ);
$output->bind("inproc://zmsg_selftest");
$input = new ZMQSocket($context, ZMQ::SOCKET_XREP);
$input->connect("inproc://zmsg_selftest");
// Test send and receive of single-part message
$zmsgo = new Zmsg($output);
$zmsgo->body_set("Hello");
$result &= assert($zmsgo->body() == "Hello");
$zmsgo->send();
$zmsgi = new Zmsg($input);
$zmsgi->recv();
$result &= assert($zmsgi->parts() == 2);
$result &= assert($zmsgi->body() == "Hello");
echo $zmsgi;
// Test send and receive of multi-part message
$zmsgo = new Zmsg($output);
$zmsgo->body_set("Hello");
$zmsgo->wrap("address1", "");
$zmsgo->wrap("address2");
$result &= assert($zmsgo->parts() == 4);
echo $zmsgo;
$zmsgo->send();
$zmsgi = new Zmsg($input);
$zmsgi->recv();
$result &= assert($zmsgi->parts() == 5);
$zmsgi->unwrap();
$result &= assert($zmsgi->unwrap() == "address2");
$zmsgi->body_fmt("%s%s", 'W', "orld");
$result &= assert($zmsgi->body() == "World");
// Pull off address 1, check that empty part was dropped
$zmsgi->unwrap();
$result &= assert($zmsgi->parts() == 1);
// Check that message body was correctly modified
$part = $zmsgi->pop();
$result &= assert($part == "World");
$result &= assert($zmsgi->parts() == 0);
// Test load and save
$zmsg = new Zmsg();
$zmsg->body_set("Hello");
$zmsg->wrap("address1", "");
$zmsg->wrap("address2");
$result &= assert($zmsg->parts() == 4);
$fh = fopen(sys_get_temp_dir() . "/zmsgtest.zmsg", 'w');
$zmsg->save($fh);
fclose($fh);
$fh = fopen(sys_get_temp_dir() . "/zmsgtest.zmsg", 'r');
$zmsg2 = new Zmsg();
$zmsg2->load($fh);
assert($zmsg2->last() == $zmsg->last());
fclose($fh);
$result &= assert($zmsg2->parts() == 4);
echo $result ? "OK" : "FAIL", PHP_EOL;
return $result;
}
/**
* Send reply, if any, to broker and wait for next request.
*
* @param Zmsg $reply
* @return Zmsg Returns if there is a request to process
*/
public function recv($reply = null)
{
// Format and send the reply if we were provided one
assert($reply || !$this->expect_reply);
if ($reply) {
$reply->wrap($this->reply_to);
$this->send_to_broker(MDPW_REPLY, NULL, $reply);
}
$this->expect_reply = true;
$read = $write = array();
while (true) {
$poll = new ZMQPoll();
$poll->add($this->worker, ZMQ::POLL_IN);
$events = $poll->poll($read, $write, $this->heartbeat);
if ($events) {
$zmsg = new Zmsg($this->worker);
$zmsg->recv();
if ($this->verbose) {
echo "I: received message from broker:", PHP_EOL;
echo $zmsg->__toString();
}
$this->liveness = HEARTBEAT_LIVENESS;
// Don't try to handle errors, just assert noisily
assert($zmsg->parts() >= 3);
$zmsg->pop();
$header = $zmsg->pop();
assert($header == MDPW_WORKER);
$command = $zmsg->pop();
if ($command == MDPW_REQUEST) {
// We should pop and save as many addresses as there are
// up to a null part, but for now, just save one...
$this->reply_to = $zmsg->unwrap();
return $zmsg;
// We have a request to process
} else {
if ($command == MDPW_HEARTBEAT) {
// Do nothing for heartbeats
} else {
if ($command == MDPW_DISCONNECT) {
$this->connect_to_broker();
} else {
echo "E: invalid input message", PHP_EOL;
echo $zmsg->__toString();
}
}
}
} else {
if (--$this->liveness == 0) {
// poll ended on timeout, $event being false
if ($this->verbose) {
echo "W: disconnected from broker - retrying...", PHP_EOL;
}
usleep($this->reconnect * 1000);
$this->connect_to_broker();
}
}
// Send HEARTBEAT if it's time
if (microtime(true) > $this->heartbeat_at) {
$this->send_to_broker(MDPW_HEARTBEAT, NULL, NULL);
$this->heartbeat_at = microtime(true) + $this->heartbeat / 1000;
}
}
}
/**
* Returns the reply message or NULL if there was no reply. Does not
* attempt to recover from a broker failure, this is not possible
* without storing all unanswered requests and resending them all...
*
*/
public function recv()
{
$read = $write = array();
// Poll socket for a reply, with timeout
$poll = new ZMQPoll();
$poll->add($this->client, ZMQ::POLL_IN);
$events = $poll->poll($read, $write, $this->timeout);
// If we got a reply, process it
if ($events) {
$msg = new Zmsg($this->client);
$msg->recv();
if ($this->verbose) {
echo "I: received reply:", $msg->__toString(), PHP_EOL;
}
// Don't try to handle errors, just assert noisily
assert($msg->parts() >= 4);
$msg->pop();
// empty
$header = $msg->pop();
assert($header == MDPC_CLIENT);
$reply_service = $msg->pop();
return $msg;
// Success
} else {
echo "W: permanent error, abandoning request", PHP_EOL;
return;
// Give up
}
}
function main()
{
for ($client_nbr = 0; $client_nbr < NBR_CLIENTS; $client_nbr++) {
$pid = pcntl_fork();
if ($pid == 0) {
client_thread();
return;
}
}
for ($worker_nbr = 0; $worker_nbr < NBR_WORKERS; $worker_nbr++) {
$pid = pcntl_fork();
if ($pid == 0) {
worker_thread();
return;
}
}
$context = new ZMQContext();
$frontend = new ZMQSocket($context, ZMQ::SOCKET_ROUTER);
$backend = new ZMQSocket($context, ZMQ::SOCKET_ROUTER);
$frontend->bind("ipc://frontend.ipc");
$backend->bind("ipc://backend.ipc");
// Logic of LRU loop
// - Poll backend always, frontend only if 1+ worker ready
// - If worker replies, queue worker as ready and forward reply
// to client if necessary
// - If client requests, pop next worker and send request to it
// Queue of available workers
$available_workers = 0;
$worker_queue = array();
$writeable = $readable = array();
while ($client_nbr > 0) {
$poll = new ZMQPoll();
// Poll front-end only if we have available workers
if ($available_workers > 0) {
$poll->add($frontend, ZMQ::POLL_IN);
}
// Always poll for worker activity on backend
$poll->add($backend, ZMQ::POLL_IN);
$events = $poll->poll($readable, $writeable);
if ($events > 0) {
foreach ($readable as $socket) {
// Handle worker activity on backend
if ($socket === $backend) {
// Queue worker address for LRU routing
$zmsg = new Zmsg($socket);
$zmsg->recv();
assert($available_workers < NBR_WORKERS);
$available_workers++;
array_push($worker_queue, $zmsg->unwrap());
if ($zmsg->body() != "READY") {
$zmsg->set_socket($frontend)->send();
// exit after all messages relayed
$client_nbr--;
}
} else {
if ($socket === $frontend) {
$zmsg = new Zmsg($socket);
$zmsg->recv();
$zmsg->wrap(array_shift($worker_queue), "");
$zmsg->set_socket($backend)->send();
$available_workers--;
}
}
}
}
}
// Clean up our worker processes
foreach ($worker_queue as $worker) {
$zmsg = new Zmsg($backend);
$zmsg->body_set('END')->wrap($worker, "")->send();
}
sleep(1);
}
function server_worker()
{
$context = new ZMQContext();
$worker = new ZMQSocket($context, ZMQ::SOCKET_DEALER);
$worker->connect("ipc://backend");
$zmsg = new Zmsg($worker);
while (true) {
// The DEALER socket gives us the address envelope and message
$zmsg->recv();
assert($zmsg->parts() == 2);
// Send 0..4 replies back
$replies = rand(0, 4);
for ($reply = 0; $reply < $replies; $reply++) {
// Sleep for some fraction of a second
usleep(rand(0, 1000) + 1);
$zmsg->send(Zmsg::NOCLEAR);
}
}
}
/**
* Send request to broker and get reply by hook or crook
* Takes ownership of request message and destroys it when sent.
* Returns the reply message or NULL if there was no reply.
*
* @param string $service
* @param Zmsg $request
* @param string $client
* @return Zmsg
*/
public function send($service, Zmsg $request)
{
// Prefix request with protocol frames
// Frame 1: "MDPCxy" (six bytes, MDP/Client
// Frame 2: Service name (printable string)
$request->push($service);
$request->push(MDPC_CLIENT);
if ($this->verbose) {
printf("I: send request to '%s' service:", $service);
echo $request->__toString();
}
$retries_left = $this->retries;
$read = $write = array();
while ($retries_left) {
$request->set_socket($this->client)->send();
// Poll socket for a reply, with timeout
$poll = new ZMQPoll();
$poll->add($this->client, ZMQ::POLL_IN);
$events = $poll->poll($read, $write, $this->timeout);
// If we got a reply, process it
if ($events) {
$request->recv();
if ($this->verbose) {
echo "I: received reply:", $request->__toString(), PHP_EOL;
}
// Don't try to handle errors, just assert noisily
assert($request->parts() >= 3);
$header = $request->pop();
assert($header == MDPC_CLIENT);
$reply_service = $request->pop();
assert($reply_service == $service);
return $request;
// Success
} elseif ($retries_left--) {
if ($this->verbose) {
echo "W: no reply, reconnecting...", PHP_EOL;
}
// Reconnect, and resend message
$this->connect_to_broker();
$request->send();
} else {
echo "W: permanent error, abandoning request", PHP_EOL;
break;
// Give up
}
}
}
protected function onPoll($events, $read, $write)
{
$events = $this->poll->poll($read, $write, $this->heartbeatDelay);
$sendHeartBeat = true;
if ($events) {
$zmsg = new Zmsg($this->socket);
$zmsg->recv();
if ($this->verbose) {
echo "I: received message from broker:", PHP_EOL;
echo $zmsg->__toString(), PHP_EOL;
}
$this->heartbeatTriesLeft = $this->heartbeatMaxFails;
$zmsg->pop();
$header = $zmsg->pop();
assert($header == Commands::W_WORKER);
$command = $zmsg->pop();
if ($command == Commands::W_HEARTBEAT) {
} elseif ($command == Commands::W_REQUEST) {
//@todo: get address
$result = call_user_func($this->executor, $zmsg->pop());
$this->send($result);
//resp = HB
$sendHeartBeat = false;
} elseif ($command == Commands::W_RESPONSE) {
$this->connect();
} else {
echo "I: Unsupported command `{$command}`.", PHP_EOL;
echo $zmsg->__toString(), PHP_EOL, PHP_EOL;
}
} elseif (--$this->heartbeatTriesLeft == 0) {
if ($this->verbose) {
echo "I: disconnected from broker - retrying... ", PHP_EOL;
}
usleep($this->reconnectDelay * 1000);
$this->connect();
}
$this->sendHeartbeat($sendHeartBeat);
}
$request_pipe->bind("ipc://" . sys_get_temp_dir() . "/titanicpipe");
$read = $write = array();
// Main dispatcher loop
while (true) {
// We'll dispatch once per second, if there's no activity
$poll = new ZMQPoll();
$poll->add($request_pipe, ZMQ::POLL_IN);
$events = $poll->poll($read, $write, 1000);
if ($events) {
// Ensure message directory exists
if (!is_dir(TITANIC_DIR)) {
mkdir(TITANIC_DIR);
}
// Append UUID to queue, prefixed with '-' for pending
$msg = new Zmsg($request_pipe);
$msg->recv();
$fh = fopen(TITANIC_DIR . "/queue", "a");
$uuid = $msg->pop();
fprintf($fh, "-%s\n", $uuid);
fclose($fh);
}
// Brute-force dispatcher
if (file_exists(TITANIC_DIR . "/queue")) {
$fh = fopen(TITANIC_DIR . "/queue", "r+");
while ($fh && ($entry = fread($fh, 33))) {
// UUID is prefixed with '-' if still waiting
if ($entry[0] == "-") {
if ($verbose) {
printf("I: processing request %s%s", substr($entry, 1), PHP_EOL);
}
if (s_service_success($client, substr($entry, 1))) {
/**
* Initialise the master server that dispatches job to the worker processes
*
* @param int $port
*/
protected function initAsMaster($port)
{
$ctx = new ZMQContext();
$client = $ctx->getSocket(ZMQ::SOCKET_XREP);
$client->bind('tcp://*:' . $port);
$workers = $ctx->getSocket(ZMQ::SOCKET_XREQ);
$workers->bind('ipc://' . $this->workersIpcName);
$this->log(__CLASS__, "Master listening on port {$port}");
$readable = $writable = array();
while (true) {
$poll = new ZMQPoll();
$poll->add($client, ZMQ::POLL_IN);
$poll->add($workers, ZMQ::POLL_IN);
$poll->poll($readable, $writable);
foreach ($readable as $socket) {
$zmsg = new Zmsg($socket);
$zmsg->recv();
if ($socket === $client) {
$this->log(__CLASS__, "Master sending job to worker");
$zmsg->set_socket($workers)->send();
} else {
if ($socket === $workers) {
$this->log(__CLASS__, "Master sending reply to client");
$zmsg->set_socket($client)->send();
}
}
}
}
}
