public function testComplex()
{
$context = new \ZMQContext();
$output = new \ZMQSocket($context, \ZMQ::SOCKET_DEALER);
$output->bind("inproc://zmsg_selftest");
$input = new \ZMQSocket($context, \ZMQ::SOCKET_ROUTER);
$input->connect("inproc://zmsg_selftest");
// Test send and receive of single-part message
$zmsgo = new Zmsg($output);
$zmsgo->setLast("Hello");
$this->assertTrue($zmsgo->getLast() == "Hello");
$zmsgo->send();
$zmsgi = new Zmsg($input);
$zmsgi->recv();
$this->assertTrue($zmsgi->parts() == 2);
$this->assertTrue($zmsgi->getLast() == "Hello");
// Test send and receive of multi-part message
$zmsgo = new Zmsg($output);
$zmsgo->setLast("Hello");
$zmsgo->wrap("address1", "");
$zmsgo->wrap("address2");
$this->assertTrue($zmsgo->parts() == 4);
$zmsgo->send();
$zmsgi = new Zmsg($input);
$zmsgi->recv();
$this->assertTrue($zmsgi->parts() == 5);
$zmsgi->unwrap();
$this->assertTrue($zmsgi->unwrap() == "address2");
$zmsgi->setLast(sprintf("%s%s", 'W', "orld"));
$this->assertTrue($zmsgi->getLast() == "World");
// Pull off address 1, check that empty part was dropped
$zmsgi->unwrap();
$this->assertTrue($zmsgi->parts() == 1);
// Check that message body was correctly modified
$part = $zmsgi->pop();
$this->assertTrue($part == "World");
$this->assertTrue($zmsgi->parts() == 0);
// Test load and save
$zmsg = new Zmsg();
$zmsg->setLast("Hello");
$zmsg->wrap("address1", "");
$zmsg->wrap("address2");
$this->assertTrue($zmsg->parts() == 4);
$fh = fopen(sys_get_temp_dir() . "/zmsgtest.zmsg", 'w');
$zmsg->save($fh);
fclose($fh);
$fh = fopen(sys_get_temp_dir() . "/zmsgtest.zmsg", 'r');
$zmsg2 = new Zmsg();
$zmsg2->load($fh);
assert($zmsg2->getLast() == $zmsg->getLast());
fclose($fh);
$this->assertTrue($zmsg2->parts() == 4);
}
/**
* Makes sure the broker is running.
* If it isn't running method will start it.
*
* @return int Returns non false if broker is running
*/
public static function ensureBrokerRunning()
{
$client = (new Client(\Scalr::config('scalr.crontab.sockets.broker')))->setTimeout(100)->setRetries(1)->setLogger(\Scalr::getContainer()->logger('Mdp\\Client')->setLevel(\Scalr::config('scalr.crontab.log_level')))->connect();
$mmiReq = new Zmsg();
$mmiReq->push("system.healthcheck");
$mmiRep = $client->send("mmi.service", $mmiReq);
if ($mmiRep) {
$ok = $mmiRep->pop();
} else {
$ok = false;
//Make sure another broker process isn't hanging
self::terminateBroker();
//Broker has to be started in the separate process
$op = [];
$logFile = \Scalr::config('scalr.crontab.log');
exec(self::getStartBrokerCommand() . ' ' . ($logFile == '/dev/null' ? '> ' : '>> ') . escapeshellcmd($logFile) . ' 2>&1 & echo $!', $op);
$pid = intval($op[0]);
}
return $ok;
}
try {
$strRequestPayload = @json_encode($payload->body);
$payload->setBody($task->worker($payload->body));
$payload->code = 200;
} catch (Exception $e) {
$task->log('ERROR', "Worker %s failed with exception:%s - %s", $task->getName(), get_class($e), $e->getMessage());
$payload = $payload->error(500, $e->getMessage());
}
$executionTime = microtime(true) - $start;
$statPath = '/var/log/scalr/worker.log';
if (is_writable($statPath)) {
@error_log(sprintf("%s,%d,\"%s\",%0.4f,%d,\"%s\"\n", date('M d H:i:s P'), isset($payload->code) ? $payload->code : 500, $service, $executionTime, \Scalr::getDb()->numberQueries + (\Scalr::getContainer()->analytics->enabled ? \Scalr::getContainer()->cadb->numberQueries : 0), str_replace('"', '""', $strRequestPayload)), 3, $statPath);
}
//Resets the number of the queries
\Scalr::getDb()->numberQueries = 0;
if (\Scalr::getContainer()->analytics->enabled) {
\Scalr::getContainer()->cadb->numberQueries = 0;
}
}
//It checks memory usage for demonized tasks
if ($config->daemon && !$task->checkMemoryUsage()) {
//Adds the pid of the process to payload to handle it on client's side
$payload->dw = posix_getpid();
//It does not even exit execution loop. Client should start a replacement in its time.
//We cannot start worker from here because it won't be correctly terminated by client.
}
$reply = new Zmsg();
$reply->setLast(serialize($payload));
unset($payload);
unset($request);
}
/**
* Send request to broker and get reply by hook or crook.
*
* Takes ownership of request message and destroys it when sent.
*
*
* @param string $service The name of the service
* @param Zmsg $request Request message
* @return Zmsg Returns the reply message or NULL if there was no reply.
*/
public function send($service, Zmsg $request)
{
// Prefix request with protocol frames
// Frame 1: "MDPCxy" (six bytes, MDP/Client)
// Frame 2: Service name (printable string)
$request->push($service);
$request->push(Mdp::CLIENT);
if ($this->verbose) {
$this->log("ZMQDEBUG", "send request to '%s' service:\n--\n%s", $service, (string) $request);
}
$retries_left = $this->retries;
$read = $write = array();
while ($retries_left) {
$request->setSocket($this->client)->send();
// Poll socket for a reply, with timeout
$poll = new ZMQPoll();
$poll->add($this->client, ZMQ::POLL_IN);
$events = $poll->poll($read, $write, $this->timeout);
// If we got a reply, process it
if ($events) {
$request->recv();
if ($this->verbose) {
$this->log("ZMQDEBUG", "received reply:\n--\n%s", $request);
}
if ($request->parts() < 3) {
throw new MdpException(sprintf("Expected more than 2 parts, but %d received", $request->parts()));
}
$header = $request->pop();
if ($header !== Mdp::CLIENT) {
throw new MdpException(sprintf("Unexpected header %s, %s is expected", $header, Mdp::CLIENT));
}
$replyService = $request->pop();
if ($replyService != $service) {
throw new MdpException(sprintf("Unexpected service %s, %s is expected.", $replyService, $service));
}
//Success
return $request;
} elseif ($retries_left--) {
if ($this->verbose) {
$this->log("WARN", "no reply, reconnecting...");
}
// Reconnect
$this->connect();
// Resend message again
$request->send();
} else {
if ($this->verbose) {
$this->log("ERROR", "permanent error, abandoning request");
break;
}
}
}
}
/**
* Receives a reply
*
* Returns the reply message or NULL if there was no reply.
* Does not attempt to recover from a broker failure, this is not possible
* without storing all unanswered requests and resending them all ...
*/
public function recv()
{
$read = $write = array();
// Poll socket for a reply, with timeout
$poll = new ZMQPoll();
$poll->add($this->client, ZMQ::POLL_IN);
$events = $poll->poll($read, $write, $this->timeout);
// If we got a reply, process it
if ($events) {
$msg = new Zmsg($this->client);
$msg->recv();
if ($this->verbose) {
$this->log('ZMQDEBUG', "received reply:\n--\n%s", (string) $msg);
}
if ($msg->parts() < 4) {
throw new MdpException("More than 3 parts are expected in the message.");
}
$msg->pop();
$header = $msg->pop();
if ($header !== Mdp::CLIENT) {
throw new MdpException(sprintf("%s is expected.", Mdp::CLIENT));
}
$repService = $msg->pop();
return $msg;
} else {
if ($this->verbose) {
$this->log('WARN', "permanent error, abandoning request");
}
return;
}
}
/**
* Send reply, if any, to broker and wait for next request.
*
* @param Zmsg $reply optional Reply message object
* @return Zmsg Returns if there is a request to process
*/
public function recv($reply = null)
{
// Format and send the reply if we were provided one
if (!$reply && $this->expectReply) {
throw new MdpException("Reply message is expected");
}
if ($reply) {
$reply->wrap($this->replyTo);
$this->send(Mdp::WORKER_REPLY, null, $reply);
}
$this->expectReply = true;
$read = $write = [];
while (true) {
$poll = new ZMQPoll();
$poll->add($this->worker, ZMQ::POLL_IN);
$events = $poll->poll($read, $write, $this->heartbeat);
if ($events) {
$zmsg = new Zmsg($this->worker);
$zmsg->recv();
if ($this->verbose) {
$this->log("ZMQDEBUG", "received message from broker:\n--\n%s", (string) $zmsg);
}
$this->liveness = self::HEARTBEAT_LIVENESS;
if ($zmsg->parts() < 3) {
throw new MdpException(sprintf("Expected more then 2 parts, but %d received", $zmsg->parts()));
}
$zmsg->pop();
$header = $zmsg->pop();
if ($header !== Mdp::WORKER) {
throw new MdpException(sprintf("Expected %s header, %s has been actually received", Mdp::WORKER, $header));
}
$command = $zmsg->pop();
if ($command == Mdp::WORKER_REQUEST) {
// We should pop and save as many addresses as there are
// up to a null part, but for now, just save one…
$this->replyTo = $zmsg->unwrap();
// We have a request to process
return $zmsg;
} elseif ($command == Mdp::WORKER_HEARTBEAT) {
// Do nothing for heartbeats
} elseif ($command == Mdp::WORKER_DISCONNECT) {
$this->connect();
} else {
if ($this->verbose) {
$this->log("ERROR", "invalid input message\n--\n%s", (string) $zmsg);
}
}
} elseif (--$this->liveness == 0) {
// poll ended on timeout, $event being false
if ($this->verbose) {
$this->log("WARN", "disconnected from broker - retrying...\n");
}
usleep($this->reconnect * 1000);
$this->connect();
}
// Send HEARTBEAT if it's time
if (microtime(true) > $this->heartbeatAt) {
$this->send(Mdp::WORKER_HEARTBEAT);
$this->updateHeartbeatExpiry();
}
}
}
/**
* Process a request coming from a client
*
* @param string $sender The address of the sender
* @param Zmsg $msg A message
*/
public function processClient($sender, Zmsg $msg)
{
$serviceFrame = $msg->pop();
$service = $this->fetchService($serviceFrame);
// Set reply return address to client sender
$msg->wrap($sender, "");
if (substr($serviceFrame, 0, 4) == 'mmi.') {
$this->handleInternalService($serviceFrame, $msg);
} else {
$this->dispatchService($service, $msg);
}
}
/**
* Runs ZMQ MDP Asynchronous Client
*
* @throws Exception
*/
protected function launchClient()
{
$this->launchWorkers();
//We don't even need to start client if queue is empty
if ($this->queue->count() == 0) {
$this->log('DEBUG', "It does not need to start major-domo client as queue is empty.");
return;
}
$this->log('DEBUG', "Launching %s 0mq mdp client", $this->name);
$session = (new AsynClient(\Scalr::config('scalr.crontab.sockets.broker'), true))->setLogger(\Scalr::getContainer()->logger('Mdp\\AsynClient')->setLevel(\Scalr::config('scalr.crontab.log_level')))->setTimeout(\Scalr::config('scalr.crontab.heartbeat.delay') * \Scalr::config('scalr.crontab.heartbeat.liveness') * 2)->connect();
$this->log('DEBUG', 'Sending request messages to broker');
$payloadClass = $this->payloadClass;
//The number of the requests sent
$count = 0;
//Array of the messages which are sent
$sentMessages = [];
//Gets queue iterator in order to gracefully iterate over an ArrayObject removing each offset
$it = $this->queue->getIterator();
//Sending messages loop
while ($it->valid()) {
$key = $it->key();
//Creates standard payload for zmq messaging
$payload = (new $payloadClass($it->current()))->setId();
$sentMessages[$payload->getId()] = $payload;
$request = new Zmsg();
$request->setLast(serialize($payload));
//Sends the message to worker
if ($payload instanceof PayloadRouterInterface) {
$session->send($payload->getAddress($this), $request);
} else {
$session->send($this->name, $request);
}
$count++;
$it->next();
//Removing the message from the queue
$it->offsetUnset($key);
}
//Cleanup queue
unset($this->queue);
$this->log('DEBUG', 'Polling results');
//Receiving loop
for ($i = 0; $i < $count; $i++) {
$msg = $session->recv();
if (!$msg) {
// Interrupt or failure
$this->getLogger()->fatal("Some worker failed!");
break;
}
//We are having deal with serialized data
$payload = @unserialize($msg->getLast());
if (!$payload instanceof AbstractPayload) {
throw new TaskException(sprintf("Unexpected reply from worker: '%s'.", $msg->getLast()));
}
//Checks if worker reaches a memory limit
if (!empty($payload->dw)) {
$this->toDisconnect[$payload->dw] = $payload instanceof PayloadRouterInterface ? $payload->getAddress($this) : $this->name;
$this->log("DEBUG", "Client got PID:%d from the worker %s to disconnect", $payload->dw, $this->toDisconnect[$payload->dw]);
}
if (!isset($sentMessages[$payload->getId()])) {
//Message comes from previous session?
$this->getLogger()->warn("Strange message came from another session. Payload:%s", var_export($payload, true));
$count++;
} else {
//We get response so remove record
unset($sentMessages[$payload->getId()]);
//Triggers onResponse callback
$this->onResponse($payload);
}
}
if (!empty($this->toDisconnect) && $this->config()->daemon) {
foreach ($this->toDisconnect as $pid => $address) {
//Terminates worker
$this->terminateWorker($pid);
//We need to get up a replacement for that one
$pid = $this->addWorker($address);
//It is important to save a PID of the process to be able terminate all workers along with client
$this->pids[$pid] = $pid;
}
//Resets event
$this->toDisconnect = [];
usleep(100000);
}
$this->onCompleted();
}
