function client_task()
{
$context = new ZMQContext();
$client = new ZMQSocket($context, ZMQ::SOCKET_DEALER);
$client->setSockOpt(ZMQ::SOCKOPT_IDENTITY, "C");
$client->connect("tcp://localhost:5555");
echo "Setting up test...", PHP_EOL;
usleep(10000);
echo "Synchronous round-trip test...", PHP_EOL;
$start = microtime(true);
$text = "HELLO";
for ($requests = 0; $requests < 10000; $requests++) {
$client->send($text);
$msg = $client->recv();
}
printf(" %d calls/second%s", 1000 * 10000 / (int) ((microtime(true) - $start) * 1000), PHP_EOL);
echo "Asynchronous round-trip test...", PHP_EOL;
$start = microtime(true);
for ($requests = 0; $requests < 100000; $requests++) {
$client->send($text);
}
for ($requests = 0; $requests < 100000; $requests++) {
$client->recv();
}
printf(" %d calls/second%s", 1000 * 100000 / (int) ((microtime(true) - $start) * 1000), PHP_EOL);
}
/**
* Notifies the task manager given a message constant, see MESSAGE_* constants.
*
* @param string $message
*
* @return mixed|null The return value of the task manager.
*
* @throws RuntimeException in case notification did not occur within the timeout.
*/
public function notify($message)
{
try {
$command = $this->createCommand($message);
$this->socket->send($command);
$result = false;
$limit = microtime(true) + $this->timeout;
while (microtime(true) < $limit && false === ($result = $this->socket->recv(\ZMQ::MODE_NOBLOCK))) {
usleep(1000);
}
if (false === $result) {
$this->logger->error(sprintf('Unable to notify the task manager with message "%s" within timeout of %d seconds', $message, $this->timeout));
throw new RuntimeException('Unable to retrieve information.');
}
$data = @json_decode($result, true);
if (JSON_ERROR_NONE !== json_last_error()) {
throw new RuntimeException('Invalid task manager response : invalid JSON.');
}
if (!isset($data['reply']) || !isset($data['request']) || $command !== $data['request']) {
throw new RuntimeException('Invalid task manager response : missing fields.');
}
return $data['reply'];
} catch (\ZMQSocketException $e) {
$this->logger->error(sprintf('Unable to notify the task manager with message "%s" within timeout of %d seconds', $message, $this->timeout), ['exception' => $e]);
throw new RuntimeException('Unable to retrieve information.', $e->getCode(), $e);
}
}
/**
* Receive message from socket
* Creates a new message and returns it
* Blocks on recv if socket is not ready for input
*
* @throws Exception if no socket present
* @return Zmsg
*/
public function recv()
{
if (!isset($this->_socket)) {
throw new Exception("No socket supplied");
}
$this->_parts = array();
while (true) {
$this->_parts[] = $this->_socket->recv();
if (!$this->_socket->getSockOpt(ZMQ::SOCKOPT_RCVMORE)) {
break;
}
}
return $this;
}
/**
* @return null
*/
public function startStackWork()
{
$getTaskDto = new ReplyStackToPulsarGetTaskRequestDto();
$considerMeAsSubscriber = 0;
while (true) {
//$this->logger->debug("Start ReplyStack while.");
$this->pulsarRequestSocket->send(serialize($getTaskDto));
/**Blocking wait reply from Pulsar
* @var PulsarToReplyStackReplyDto $pulsarToReplyStackReplyDto
*/
$pulsarToReplyStackReplyDto = unserialize($this->pulsarRequestSocket->recv());
//$this->logger->debug("REPLY STACK asked to prepare subscribers: " . $pulsarToReplyStackReplyDto->getSubscribersNumber());
for ($i = 1; $i <= $pulsarToReplyStackReplyDto->getSubscribersNumber(); $i++) {
$preparingDto = unserialize($this->performersReplySocket->recv());
//$this->logger->debug("REPLY STACK: receive request $i");
if ($preparingDto instanceof PreparingRequestDto) {
$considerMeAsSubscriber++;
//$this->logger->debug("REPLY STACK: considerMeAsSubscriber: $considerMeAsSubscriber");
$this->performersReplySocket->send(serialize($pulsarToReplyStackReplyDto->getDtoToTransfer()));
}
}
//$this->logger->debug("REPLY STACK prepared subscribers: " . $considerMeAsSubscriber);
$replyStackResult = new ReplyStackToPulsarReturnResultRequestDto();
$replyStackResult->setConsiderMeAsSubscriber($considerMeAsSubscriber);
$this->pulsarRequestSocket->send(serialize($replyStackResult));
//$this->logger->debug("Wait finishing message from Pulsar.");
$this->pulsarRequestSocket->recv();
//$this->logger->debug("Got finish message from Pulsar.");
$considerMeAsSubscriber = 0;
//$this->logger->debug("Finish ReplyStack while.");
}
return null;
}
/**
* Start the worker and wait for requests
*/
public function listen()
{
$context = new \ZMQContext();
$server = new \ZMQSocket($context, \ZMQ::SOCKET_PULL);
$server->bind('tcp://127.0.0.1:' . ($this->defaultPort + $this->client->getId() - 1));
$this->logger->info('Client worker ' . $this->client . ' is ready');
while (true) {
$request = $server->recv();
$this->logger->debug('Client worker ' . $this->client . ' receiving request : ' . $request);
// Check if the input is valid, ignore if wrong
$request = json_decode($request, true);
if (!$this->isValidInput($request)) {
$this->logger->error('Client worker ' . $this->client . ' received an invalid input');
continue;
}
try {
// Call the right method in the client and push to redis the result
$result = call_user_func_array(array($this->client, $request['command']), $request['parameters']);
} catch (ClientNotReadyException $e) {
$this->logger->warning('Client worker ' . $this->client . ' received a request (#' . $request['invokeId'] . ') whereas the client is not ready. This is normal in case of client reconnection process. Ignoring.');
continue;
}
$key = $this->key . '.client.commands.' . $request['invokeId'];
$this->redis->rpush($key, serialize($result));
$this->redis->expire($key, $this->expire);
}
}
public function collect(OutputInterface $output)
{
$context = new \ZMQContext();
$resultsQueue = new \ZMQSocket($context, \ZMQ::SOCKET_PULL);
$resultsQueue->bind(Spider::ZMQ_RESULTS_QUEUE_BIND_DSN);
$statusQueue = new \ZMQSocket($context, \ZMQ::SOCKET_PUSH);
$statusQueue->bind(Spider::ZMQ_STATUS_QUEUE_BIND_DSN);
$tstart = microtime(true);
$collectedResults = 0;
$expectedResults = PHP_INT_MAX;
$output->writeln('Collecting Task results');
while ($collectedResults < $expectedResults) {
$string = $resultsQueue->recv();
if ($string === Spider::ZMQ_COMMAND_BATCH_START) {
// Wait for start of batch
} elseif (stripos($string, Spider::ZMQ_COMMAND_BATCH_END) === false) {
$output->writeln('Got task result: ' . substr($string, 0, 20) . ' ...');
file_put_contents($this->resultsTargetPath . '/' . md5($string) . '.result', $string);
// TODO: use Symfony/Filesystem
$output->writeln('Collected results so far: ' . ++$collectedResults);
} else {
$expectedResults = (int) explode('%', $string)[1];
$output->writeln('[INFO] Trying to collect ' . $expectedResults . ' as requested by Task Loader');
}
}
$tend = microtime(true);
$totalMsec = ($tend - $tstart) * 1000;
$output->writeln('Task results collecting finished. Got ' . $collectedResults . ' results');
$output->writeln("Total elapsed time: {$totalMsec} msec");
$output->writeln('Sending Task Result Collector info');
$statusQueue->send($collectedResults);
}
/**
* Run ZMQ interface for generator
*
* Req-rep pattern; msgs are commands:
*
* GEN = Generate ID
* STATUS = Get status string
*/
public function run()
{
$context = new \ZMQContext();
$receiver = new \ZMQSocket($context, \ZMQ::SOCKET_REP);
$bindTo = 'tcp://*:' . $this->port;
echo "Binding to {$bindTo}\n";
$receiver->bind($bindTo);
while (TRUE) {
$msg = $receiver->recv();
switch ($msg) {
case 'GEN':
try {
$response = $this->generator->generate();
} catch (\Exception $e) {
$response = "ERROR";
}
break;
case 'STATUS':
$response = json_encode($this->generator->status());
break;
default:
$response = 'UNKNOWN COMMAND';
break;
}
$receiver->send($response);
}
}
public function run(OutputInterface $output)
{
$context = new \ZMQContext();
$tasksQueue = new \ZMQSocket($context, \ZMQ::SOCKET_PULL);
$tasksQueue->connect(Spider::ZMQ_TASKS_QUEUE_DSN);
$resultsQueue = new \ZMQSocket($context, \ZMQ::SOCKET_PUSH);
$resultsQueue->connect(Spider::ZMQ_RESULTS_QUEUE_DSN);
$output->writeln('HTTP Worker is waiting for tasks');
while (true) {
$string = $tasksQueue->recv();
if ($string === Spider::ZMQ_COMMAND_BATCH_START) {
$resultsQueue->send($string);
} elseif (stripos($string, Spider::ZMQ_COMMAND_BATCH_END) !== false) {
// send info for result collector how many results it should expect
$resultsQueue->send($string);
} elseif ($string === Spider::ZMQ_COMMAND_WORKER_QUIT) {
$output->writeln('No more work. Worker stops now.');
break;
// no more work
} else {
$output->writeln('Fetching data from URI: ' . $string);
$userData = file_get_contents($string);
// TODO: use Guzzle
$output->writeln('Sending result for URI: ' . $string);
$resultsQueue->send($userData);
}
}
}
/**
* @param string|null $queueId
* @param int $waitTime
*
* @return QueueMessage
*/
public function receiveMessage($queueId = null, $waitTime = null)
{
$queueId = empty($queueId) ? $this->getQueueId() : $queueId;
$this->setupPullSocket($queueId);
$this->pull->setSockOpt(\ZMQ::SOCKOPT_RCVTIMEO, isset($waitTime) ? $waitTime : $this->waitTime);
$message = $this->pull->recv();
return $this->messageFactory->createMessage($message, $queueId);
}
function client_thread()
{
$context = new ZMQContext();
$client = new ZMQSocket($context, ZMQ::SOCKET_REQ);
$client->connect("ipc://frontend.ipc");
// Send request, get reply
$client->send("HELLO");
$reply = $client->recv();
printf("Client: %s%s", $reply, PHP_EOL);
}
private function checkForExternalCommand()
{
$cmd = $this->zmq_socket->recv(\ZMQ::MODE_NOBLOCK);
if ($cmd != null) {
switch ($cmd) {
case 'refresh jobs':
$this->jobs = $this->schedule->getAllJobs();
$this->zmq_socket->send(1, \ZMQ::MODE_NOBLOCK);
break;
case 'get loaded jobs':
$data = array();
foreach ($this->jobs as $job) {
$data[] = (array) $job;
}
$this->zmq_socket->send(serialize($data), \ZMQ::MODE_NOBLOCK);
break;
}
}
}
function url_hash_worker()
{
// Socket to talk to dispatcher
$context = new ZMQContext();
$receiver = new ZMQSocket($context, ZMQ::SOCKET_REP);
$receiver->connect("ipc://urlhash.ipc");
while (true) {
$hashedUrl = $receiver->recv();
$receiver->send(hashDecode($hashedUrl));
}
}
function client_thread($self)
{
$context = new ZMQContext();
$client = new ZMQSocket($context, ZMQ::SOCKET_REQ);
$endpoint = sprintf("ipc://%s-localfe.ipc", $self);
$client->connect($endpoint);
while (true) {
// Send request, get reply
$client->send("HELLO");
$reply = $client->recv();
printf("I: client status: %s%s", $reply, PHP_EOL);
}
}
function __call($name, $arguments)
{
$context = new \ZMQContext();
$zmq = new \ZMQSocket($context, \ZMQ::SOCKET_REQ);
$zmq->connect($this->socket);
$message = array('jsonrpc' => '2.0', 'method' => $name, 'params' => $arguments, 'id' => round(microtime(true) * 100000));
/** @var \ZMQSocket $resultSocket */
$zmq->send(json_encode($message));
$result = json_decode($zmq->recv(), true);
if (isset($result['error'])) {
throw new RpcException($result['exception']['message'], $result['exception']['code']);
}
return $result['result'];
}
function worker_routine()
{
$context = new ZMQContext();
// Socket to talk to dispatcher
$receiver = new ZMQSocket($context, ZMQ::SOCKET_REP);
$receiver->connect("ipc://workers.ipc");
while (true) {
$string = $receiver->recv();
printf("Received request: [%s]%s", $string, PHP_EOL);
// Do some 'work'
sleep(1);
// Send reply back to client
$receiver->send("World");
}
}
/**
* Starts an endless running service
*/
public function start()
{
$context = new \ZMQContext();
$zmq = new \ZMQSocket($context, \ZMQ::SOCKET_REP);
$zmq->bind($this->socket);
while (true) {
try {
$message = '' . $zmq->recv();
$object = json_decode($message, true);
$method = $object['method'];
$args = $object['params'];
$id = $object['id'];
$response = $this->handleRequest($id, $method, $args);
$zmq->send(json_encode($response));
} catch (\Exception $e) {
}
}
}
function step2()
{
$pid = pcntl_fork();
if ($pid == 0) {
step1();
exit;
}
$context = new ZMQContext();
// Bind to ipc: endpoint, then start upstream thread
$receiver = new ZMQSocket($context, ZMQ::SOCKET_PAIR);
$receiver->bind("ipc://step2.ipc");
// Wait for signal
$receiver->recv();
// Signal downstream to step 3
$sender = new ZMQSocket($context, ZMQ::SOCKET_PAIR);
$sender->connect("ipc://step3.ipc");
$sender->send("");
}
function subscriber()
{
$context = new ZMQContext();
// Subscribe to everything
$subscriber = new ZMQSocket($context, ZMQ::SOCKET_SUB);
$subscriber->connect("tcp://localhost:5556");
$subscriber->setSockOpt(ZMQ::SOCKOPT_SUBSCRIBE, "");
// Get and process messages
while (true) {
$clock = $subscriber->recv();
// Suicide snail logic
if (microtime(true) * 100 - $clock * 100 > MAX_ALLOWED_DELAY) {
echo "E: subscriber cannot keep up, aborting", PHP_EOL;
break;
}
// Work for 1 msec plus some random additional time
usleep(1000 + rand(0, 1000));
}
}
public function waitAllowingSubscriptionMessage()
{
if (!$this->subscriberSocket) {
$this->subscriberSocket = $this->context->getSocket(\ZMQ::SOCKET_SUB);
$this->subscriberSocket->connect($this->getSocketsParams()->getPublisherPulsarSocketAddress());
}
$this->subscriberSocket->setSockOpt(\ZMQ::SOCKOPT_SUBSCRIBE, "");
$this->logger->debug("Performer become subscriber.");
//use in Pulsar to divide termination of current subscribers and potential subscribers, if early termination occurred
$this->performerEarlyTerminated->setStandOnSubscription(true);
$this->pushReadyToGetSubscriptionMsg();
//->recv() block execution until data will received; all subscribers will continue execution synchronously
$this->logger->debug("Performer wait subscription msg.");
$this->publisherToSubscribersDto = unserialize($this->subscriberSocket->recv());
$this->logger->debug("Performer got subscription msg.");
if (!$this->publisherToSubscribersDto->isAllowAction()) {
throw new PublisherPulsarException(PublisherPulsarExceptionsConstants::ACTION_IS_NOT_ALLOWED);
}
return null;
}
function worker_thread()
{
$context = new ZMQContext();
$worker = new ZMQSocket($context, ZMQ::SOCKET_REQ);
$worker->connect("ipc://routing.ipc");
$total = 0;
while (true) {
// Tell the router we're ready for work
$worker->send("ready");
// Get workload from router, until finished
$workload = $worker->recv();
if ($workload == 'END') {
printf("Processed: %d tasks%s", $total, PHP_EOL);
break;
}
$total++;
// Do some random work
usleep(mt_rand(1, 1000000));
}
}
public function load($inputStream, OutputInterface $output)
{
$context = new \ZMQContext();
$tasksQueue = new \ZMQSocket($context, \ZMQ::SOCKET_PUSH);
$tasksQueue->bind(Spider::ZMQ_TASKS_QUEUE_BIND_DSN);
$statusQueue = new \ZMQSocket($context, \ZMQ::SOCKET_PULL);
$statusQueue->connect(Spider::ZMQ_STATUS_QUEUE_DSN);
/*
* http://zguide.zeromq.org/php:all#advanced-request-reply
* We have to synchronize the start of the batch with all workers being up and running.
* This is a fairly common gotcha in ZeroMQ and there is no easy solution.
* The zmq_connect method takes a certain time.
* So when a set of workers connect to the ventilator, the first one to successfully connect will get a whole load of messages
* in that short time while the others are also connecting.
* If you don't synchronize the start of the batch somehow, the system won't run in parallel at all.
* Try removing the wait in the ventilator, and see what happens.
*/
$output->writeln('Giving workers some time to connect');
sleep(3);
// The first message is "BATCH_START%" and signals start of batch
$tasksQueue->send(Spider::ZMQ_COMMAND_BATCH_START);
$taskCount = 0;
while (($task = fgets($inputStream)) !== false) {
$task = trim(preg_replace('/\\s\\s+/', ' ', $task));
$tasksQueue->send($task);
++$taskCount;
}
$tasksQueue->send(Spider::ZMQ_COMMAND_BATCH_END . $taskCount);
// send info for result collector how many results it should expect
$output->writeln("<info>Total count of Tasks put in the Queue: {$taskCount}</info>");
sleep(1);
// Give 0MQ time to deliver
$output->writeln('Waiting for acknowledgement from Task Result Collector');
$output->writeln('Info from Task Result Collector: ' . $statusQueue->recv());
$output->writeln('<info>Informing all workers to stop</info>');
for ($i = 0; $i < 10; ++$i) {
$tasksQueue->send(Spider::ZMQ_COMMAND_WORKER_QUIT);
}
}
<?php
/**
* Created by PhpStorm.
* User: henrygrech-cini
* Date: 30/10/2014
* Time: 22:48
*/
// A very simple HTTP to ZMQ Proxy
// This could be setup as an asynchronous proxy
// but currently we wait for the response.
// The socket can also be invoked externally
// port 5454 has been opened on the AWS EC2 instance
// code to show the code
echo 'Code shown here to illustrate' . PHP_EOL;
echo show_source(__FILE__);
// the actual code
// create the ZMQ Context and Socket
$ctx = new ZMQContext();
$req = new ZMQSocket($ctx, ZMQ::SOCKET_REQ);
// connect to the ZMQ
$req->connect('tcp://localhost:5454');
// prepare the message to send to ZMQ
$message = isset($_GET['msg']) ? $_GET['msg'] : 'Default Hello';
// send the prepared message
$req->send($message);
// output the response
echo PHP_EOL . '[ ' . $req->recv() . ' ]' . PHP_EOL;
// Actual Response SHOW BELOW //
/*
* Lazy Pirate server
* Binds REQ socket to tcp://*:5555
* Like hwserver except:
* - echoes request as-is
* - randomly runs slowly, or exits to simulate a crash.
*
* @author Ian Barber <ian(dot)barber(at)gmail(dot)com>
*/
$context = new ZMQContext();
$server = new ZMQSocket($context, ZMQ::SOCKET_REP);
$server->bind("tcp://*:5555");
$cycles = 0;
while (true) {
$request = $server->recv();
$cycles++;
// Simulate various problems, after a few cycles
if ($cycles > 3 && rand(0, 3) == 0) {
echo "I: simulating a crash", PHP_EOL;
break;
} else {
if ($cycles > 3 && rand(0, 3) == 0) {
echo "I: simulating CPU overload", PHP_EOL;
sleep(5);
}
}
printf("I: normal request (%s)%s", $request, PHP_EOL);
sleep(1);
// Do some heavy work
$server->send($request);
<?php
/**
* Created by PhpStorm.
* User: henrygrech-cini
* Date: 30/10/2014
* Time: 22:46
*/
$ctx = new ZMQContext();
$server = new ZMQSocket($ctx, ZMQ::SOCKET_REP);
$server->bind('tcp://*:5454');
while (true) {
$message = $server->recv();
$server->send($message . ' World');
}
<?php
/*
* Hello World client
* Connects REQ socket to tcp://localhost:5555
* Sends "Hello" to server, expects "World" back
* @author Ian Barber <ian(dot)barber(at)gmail(dot)com>
*/
$context = new ZMQContext();
// Socket to talk to server
echo "Connecting to server.\n";
$requester = new ZMQSocket($context, ZMQ::SOCKET_REQ);
$requester->connect("ipc:///var/www/socks/buy.ipc");
$requester->send("{\"user_id\":1,\"item_id\":2}");
$reply = $requester->recv();
printf("Received reply [%s]\n", $reply);
<?php
/*
* Durable subscriber
* @author Ian Barber <ian(dot)barber(at)gmail(dot)com>
*/
$context = new ZMQContext(1);
// Connect our subscriber socket
$subscriber = new ZMQSocket($context, ZMQ::SOCKET_SUB);
$subscriber->setSockOpt(ZMQ::SOCKOPT_IDENTITY, "Hello");
$subscriber->setSockOpt(ZMQ::SOCKOPT_SUBSCRIBE, "");
$subscriber->connect("tcp://localhost:5565");
// Synchronize with publisher
$sync = new ZMQSocket($context, ZMQ::SOCKET_PUSH);
$sync->connect("tcp://localhost:5564");
$sync->send("");
// Get updates, expect random Ctrl-C death
while (true) {
$string = $subscriber->recv();
echo $string, "\n";
if ($string == "END") {
break;
}
}
<?php
$context = new ZMQContext();
// Socket to talk to server
echo "Collecting updates from weather server...", PHP_EOL;
$subscriber = new ZMQSocket($context, ZMQ::SOCKET_SUB);
$subscriber->connect("tcp://localhost:5556");
$subscriber->setSockOpt(ZMQ::SOCKOPT_SUBSCRIBE, 'Message');
// Process 100 updates
while (true) {
print $subscriber->recv();
}
<?php
/*
* Task sink
* Binds PULL socket to tcp://localhost:5558
* Collects results from workers via that socket
* @author Ian Barber <ian(dot)barber(at)gmail(dot)com>
*/
// Prepare our context and socket
$context = new ZMQContext();
$receiver = new ZMQSocket($context, ZMQ::SOCKET_PULL);
$receiver->bind("tcp://*:5558");
// Wait for start of batch
$string = $receiver->recv();
// Start our clock now
$tstart = microtime(true);
// Process 100 confirmations
$total_msec = 0;
// Total calculated cost in msecs
for ($task_nbr = 0; $task_nbr < 100; $task_nbr++) {
$string = $receiver->recv();
if ($task_nbr % 10 == 0) {
echo ":";
} else {
echo ".";
}
}
$tend = microtime(true);
$total_msec = ($tend - $tstart) * 1000;
echo PHP_EOL;
printf("Total elapsed time: %d msec", $total_msec);
<?php
$context = new ZMQContext();
$receiver = new ZMQSocket($context, ZMQ::SOCKET_PAIR);
$receiver->bind("tcp://127.0.0.1:5557");
$receiver->recv();
echo "Получил сообщение", PHP_EOL;
$sender = new ZMQSocket($context, ZMQ::SOCKET_PAIR);
$sender->connect("tcp://127.0.0.1:5558");
$sender->send("");
echo "Отправил сообщение", PHP_EOL;
<?php
/*
* Hello World server
* Binds REP socket to tcp://*:5555
* Expects "Hello" from client, replies with "World"
* @author Ian Barber <ian(dot)barber(at)gmail(dot)com>
*/
$context = new ZMQContext(1);
// Socket to talk to clients
$responder = new ZMQSocket($context, ZMQ::SOCKET_REP);
$responder->bind("tcp://*:5555");
while (true) {
// Wait for next request from client
$request = $responder->recv();
printf("Received request: [%s]\n", $request);
// Do some 'work'
sleep(1);
// Send reply back to client
$responder->send("World");
}
