function client_thread($self)
{
$context = new ZMQContext();
$client = new ZMQSocket($context, ZMQ::SOCKET_REQ);
$endpoint = sprintf("ipc://%s-localfe.ipc", $self);
$client->connect($endpoint);
$monitor = new ZMQSocket($context, ZMQ::SOCKET_PUSH);
$endpoint = sprintf("ipc://%s-monitor.ipc", $self);
$monitor->connect($endpoint);
$readable = $writeable = array();
while (true) {
sleep(mt_rand(0, 4));
$burst = mt_rand(1, 14);
while ($burst--) {
// Send request with random hex ID
$task_id = sprintf("%04X", mt_rand(0, 10000));
$client->send($task_id);
// Wait max ten seconds for a reply, then complain
$poll = new ZMQPoll();
$poll->add($client, ZMQ::POLL_IN);
$events = $poll->poll($readable, $writeable, 10 * 1000000);
if ($events > 0) {
foreach ($readable as $socket) {
$zmsg = new Zmsg($socket);
$zmsg->recv();
// Worker is supposed to answer us with our task id
assert($zmsg->body() == $task_id);
}
} else {
$monitor->send(sprintf("E: CLIENT EXIT - lost task %s", $task_id));
exit;
}
}
}
}
function client_task()
{
$context = new ZMQContext();
$client = new ZMQSocket($context, ZMQ::SOCKET_DEALER);
$client->setSockOpt(ZMQ::SOCKOPT_IDENTITY, "C");
$client->connect("tcp://localhost:5555");
echo "Setting up test...", PHP_EOL;
usleep(10000);
echo "Synchronous round-trip test...", PHP_EOL;
$start = microtime(true);
$text = "HELLO";
for ($requests = 0; $requests < 10000; $requests++) {
$client->send($text);
$msg = $client->recv();
}
printf(" %d calls/second%s", 1000 * 10000 / (int) ((microtime(true) - $start) * 1000), PHP_EOL);
echo "Asynchronous round-trip test...", PHP_EOL;
$start = microtime(true);
for ($requests = 0; $requests < 100000; $requests++) {
$client->send($text);
}
for ($requests = 0; $requests < 100000; $requests++) {
$client->recv();
}
printf(" %d calls/second%s", 1000 * 100000 / (int) ((microtime(true) - $start) * 1000), PHP_EOL);
}
/**
* @param string $message
* @return mixed
*/
public function send($message)
{
if ($this->isVerbose()) {
$this->logger->debug("Sending message: " . $message);
}
$this->socket->send($message);
}
/**
* Notifies the task manager given a message constant, see MESSAGE_* constants.
*
* @param string $message
*
* @return mixed|null The return value of the task manager.
*
* @throws RuntimeException in case notification did not occur within the timeout.
*/
public function notify($message)
{
try {
$command = $this->createCommand($message);
$this->socket->send($command);
$result = false;
$limit = microtime(true) + $this->timeout;
while (microtime(true) < $limit && false === ($result = $this->socket->recv(\ZMQ::MODE_NOBLOCK))) {
usleep(1000);
}
if (false === $result) {
$this->logger->error(sprintf('Unable to notify the task manager with message "%s" within timeout of %d seconds', $message, $this->timeout));
throw new RuntimeException('Unable to retrieve information.');
}
$data = @json_decode($result, true);
if (JSON_ERROR_NONE !== json_last_error()) {
throw new RuntimeException('Invalid task manager response : invalid JSON.');
}
if (!isset($data['reply']) || !isset($data['request']) || $command !== $data['request']) {
throw new RuntimeException('Invalid task manager response : missing fields.');
}
return $data['reply'];
} catch (\ZMQSocketException $e) {
$this->logger->error(sprintf('Unable to notify the task manager with message "%s" within timeout of %d seconds', $message, $this->timeout), ['exception' => $e]);
throw new RuntimeException('Unable to retrieve information.', $e->getCode(), $e);
}
}
public function run(OutputInterface $output)
{
$context = new \ZMQContext();
$tasksQueue = new \ZMQSocket($context, \ZMQ::SOCKET_PULL);
$tasksQueue->connect(Spider::ZMQ_TASKS_QUEUE_DSN);
$resultsQueue = new \ZMQSocket($context, \ZMQ::SOCKET_PUSH);
$resultsQueue->connect(Spider::ZMQ_RESULTS_QUEUE_DSN);
$output->writeln('HTTP Worker is waiting for tasks');
while (true) {
$string = $tasksQueue->recv();
if ($string === Spider::ZMQ_COMMAND_BATCH_START) {
$resultsQueue->send($string);
} elseif (stripos($string, Spider::ZMQ_COMMAND_BATCH_END) !== false) {
// send info for result collector how many results it should expect
$resultsQueue->send($string);
} elseif ($string === Spider::ZMQ_COMMAND_WORKER_QUIT) {
$output->writeln('No more work. Worker stops now.');
break;
// no more work
} else {
$output->writeln('Fetching data from URI: ' . $string);
$userData = file_get_contents($string);
// TODO: use Guzzle
$output->writeln('Sending result for URI: ' . $string);
$resultsQueue->send($userData);
}
}
}
/**
* {@inheritDoc}
*/
protected function write(array $record)
{
if ($this->multipart) {
$this->zmqSocket->send($record['channel'], $this->zmqMode);
$this->zmqSocket->send($record['formatted']);
} else {
$this->zmqSocket->send($record["formatted"], $this->zmqMode);
}
}
/**
* @param string|array $message
* @param int $mode
*/
public function send($message, $mode = 0)
{
if (false === $this->connected) {
$connectedTo = $this->socket->getEndpoints();
if (!in_array($this->dsn, $connectedTo)) {
$this->socket->connect($this->dsn);
}
$this->connected = true;
}
$this->socket->send($message, $mode);
}
/**
* Sends message to socket.
*
* @param boolean $clear optional Whether it shoud destroy message parts
* @return Zmsg
* @throws \Exception
*/
public function send($clear = true)
{
if (!isset($this->socket)) {
throw new \Exception("No socket supplied");
}
$count = count($this->parts);
foreach ($this->parts as $part) {
$this->socket->send($part, --$count ? \ZMQ::MODE_SNDMORE : null);
}
if ($clear) {
unset($this->parts);
$this->parts = [];
}
return $this;
}
/**
* @return null
*/
public function pushReadyToGetSubscriptionMsg()
{
$this->initOrCheckPushConnection();
$this->pushSocket->send(serialize(new ReadyToGetSubscriptionMsg()));
$this->logger->debug("Performer send that ready to get subscription msg.");
return null;
}
/**
* @return null
*/
public function startStackWork()
{
$getTaskDto = new ReplyStackToPulsarGetTaskRequestDto();
$considerMeAsSubscriber = 0;
while (true) {
//$this->logger->debug("Start ReplyStack while.");
$this->pulsarRequestSocket->send(serialize($getTaskDto));
/**Blocking wait reply from Pulsar
* @var PulsarToReplyStackReplyDto $pulsarToReplyStackReplyDto
*/
$pulsarToReplyStackReplyDto = unserialize($this->pulsarRequestSocket->recv());
//$this->logger->debug("REPLY STACK asked to prepare subscribers: " . $pulsarToReplyStackReplyDto->getSubscribersNumber());
for ($i = 1; $i <= $pulsarToReplyStackReplyDto->getSubscribersNumber(); $i++) {
$preparingDto = unserialize($this->performersReplySocket->recv());
//$this->logger->debug("REPLY STACK: receive request $i");
if ($preparingDto instanceof PreparingRequestDto) {
$considerMeAsSubscriber++;
//$this->logger->debug("REPLY STACK: considerMeAsSubscriber: $considerMeAsSubscriber");
$this->performersReplySocket->send(serialize($pulsarToReplyStackReplyDto->getDtoToTransfer()));
}
}
//$this->logger->debug("REPLY STACK prepared subscribers: " . $considerMeAsSubscriber);
$replyStackResult = new ReplyStackToPulsarReturnResultRequestDto();
$replyStackResult->setConsiderMeAsSubscriber($considerMeAsSubscriber);
$this->pulsarRequestSocket->send(serialize($replyStackResult));
//$this->logger->debug("Wait finishing message from Pulsar.");
$this->pulsarRequestSocket->recv();
//$this->logger->debug("Got finish message from Pulsar.");
$considerMeAsSubscriber = 0;
//$this->logger->debug("Finish ReplyStack while.");
}
return null;
}
public function collect(OutputInterface $output)
{
$context = new \ZMQContext();
$resultsQueue = new \ZMQSocket($context, \ZMQ::SOCKET_PULL);
$resultsQueue->bind(Spider::ZMQ_RESULTS_QUEUE_BIND_DSN);
$statusQueue = new \ZMQSocket($context, \ZMQ::SOCKET_PUSH);
$statusQueue->bind(Spider::ZMQ_STATUS_QUEUE_BIND_DSN);
$tstart = microtime(true);
$collectedResults = 0;
$expectedResults = PHP_INT_MAX;
$output->writeln('Collecting Task results');
while ($collectedResults < $expectedResults) {
$string = $resultsQueue->recv();
if ($string === Spider::ZMQ_COMMAND_BATCH_START) {
// Wait for start of batch
} elseif (stripos($string, Spider::ZMQ_COMMAND_BATCH_END) === false) {
$output->writeln('Got task result: ' . substr($string, 0, 20) . ' ...');
file_put_contents($this->resultsTargetPath . '/' . md5($string) . '.result', $string);
// TODO: use Symfony/Filesystem
$output->writeln('Collected results so far: ' . ++$collectedResults);
} else {
$expectedResults = (int) explode('%', $string)[1];
$output->writeln('[INFO] Trying to collect ' . $expectedResults . ' as requested by Task Loader');
}
}
$tend = microtime(true);
$totalMsec = ($tend - $tstart) * 1000;
$output->writeln('Task results collecting finished. Got ' . $collectedResults . ' results');
$output->writeln("Total elapsed time: {$totalMsec} msec");
$output->writeln('Sending Task Result Collector info');
$statusQueue->send($collectedResults);
}
/**
* Run ZMQ interface for generator
*
* Req-rep pattern; msgs are commands:
*
* GEN = Generate ID
* STATUS = Get status string
*/
public function run()
{
$context = new \ZMQContext();
$receiver = new \ZMQSocket($context, \ZMQ::SOCKET_REP);
$bindTo = 'tcp://*:' . $this->port;
echo "Binding to {$bindTo}\n";
$receiver->bind($bindTo);
while (TRUE) {
$msg = $receiver->recv();
switch ($msg) {
case 'GEN':
try {
$response = $this->generator->generate();
} catch (\Exception $e) {
$response = "ERROR";
}
break;
case 'STATUS':
$response = json_encode($this->generator->status());
break;
default:
$response = 'UNKNOWN COMMAND';
break;
}
$receiver->send($response);
}
}
private function checkForExternalCommand()
{
$cmd = $this->zmq_socket->recv(\ZMQ::MODE_NOBLOCK);
if ($cmd != null) {
switch ($cmd) {
case 'refresh jobs':
$this->jobs = $this->schedule->getAllJobs();
$this->zmq_socket->send(1, \ZMQ::MODE_NOBLOCK);
break;
case 'get loaded jobs':
$data = array();
foreach ($this->jobs as $job) {
$data[] = (array) $job;
}
$this->zmq_socket->send(serialize($data), \ZMQ::MODE_NOBLOCK);
break;
}
}
}
/**
* @param string $commandName
* @param array $parameters
* @param int|string|null $invokeId
*
* @return int|string
*/
protected function send($commandName, array $parameters = array(), $invokeId = null)
{
if (null == $invokeId) {
$invokeId = $this->redis->incr($this->getKey('invokeId'));
}
$nextAvailableTime = (double) ConfigurationLoader::get('client.request.overload.available');
$this->lastCall = microtime(true) + $nextAvailableTime;
$this->con->send(json_encode(['invokeId' => $invokeId, 'command' => $commandName, 'parameters' => $parameters]), \ZMQ::MODE_DONTWAIT);
return $invokeId;
}
function client_thread()
{
$context = new ZMQContext();
$client = new ZMQSocket($context, ZMQ::SOCKET_REQ);
$client->connect("ipc://frontend.ipc");
// Send request, get reply
$client->send("HELLO");
$reply = $client->recv();
printf("Client: %s%s", $reply, PHP_EOL);
}
function url_hash_worker()
{
// Socket to talk to dispatcher
$context = new ZMQContext();
$receiver = new ZMQSocket($context, ZMQ::SOCKET_REP);
$receiver->connect("ipc://urlhash.ipc");
while (true) {
$hashedUrl = $receiver->recv();
$receiver->send(hashDecode($hashedUrl));
}
}
public function load($inputStream, OutputInterface $output)
{
$context = new \ZMQContext();
$tasksQueue = new \ZMQSocket($context, \ZMQ::SOCKET_PUSH);
$tasksQueue->bind(Spider::ZMQ_TASKS_QUEUE_BIND_DSN);
$statusQueue = new \ZMQSocket($context, \ZMQ::SOCKET_PULL);
$statusQueue->connect(Spider::ZMQ_STATUS_QUEUE_DSN);
/*
* http://zguide.zeromq.org/php:all#advanced-request-reply
* We have to synchronize the start of the batch with all workers being up and running.
* This is a fairly common gotcha in ZeroMQ and there is no easy solution.
* The zmq_connect method takes a certain time.
* So when a set of workers connect to the ventilator, the first one to successfully connect will get a whole load of messages
* in that short time while the others are also connecting.
* If you don't synchronize the start of the batch somehow, the system won't run in parallel at all.
* Try removing the wait in the ventilator, and see what happens.
*/
$output->writeln('Giving workers some time to connect');
sleep(3);
// The first message is "BATCH_START%" and signals start of batch
$tasksQueue->send(Spider::ZMQ_COMMAND_BATCH_START);
$taskCount = 0;
while (($task = fgets($inputStream)) !== false) {
$task = trim(preg_replace('/\\s\\s+/', ' ', $task));
$tasksQueue->send($task);
++$taskCount;
}
$tasksQueue->send(Spider::ZMQ_COMMAND_BATCH_END . $taskCount);
// send info for result collector how many results it should expect
$output->writeln("<info>Total count of Tasks put in the Queue: {$taskCount}</info>");
sleep(1);
// Give 0MQ time to deliver
$output->writeln('Waiting for acknowledgement from Task Result Collector');
$output->writeln('Info from Task Result Collector: ' . $statusQueue->recv());
$output->writeln('<info>Informing all workers to stop</info>');
for ($i = 0; $i < 10; ++$i) {
$tasksQueue->send(Spider::ZMQ_COMMAND_WORKER_QUIT);
}
}
function publisher()
{
$context = new ZMQContext();
// Prepare publisher
$publisher = new ZMQSocket($context, ZMQ::SOCKET_PUB);
$publisher->bind("tcp://*:5556");
while (true) {
// Send current clock (msecs) to subscribers
$publisher->send(microtime(true));
usleep(1000);
// 1msec wait
}
}
function client_thread($self)
{
$context = new ZMQContext();
$client = new ZMQSocket($context, ZMQ::SOCKET_REQ);
$endpoint = sprintf("ipc://%s-localfe.ipc", $self);
$client->connect($endpoint);
while (true) {
// Send request, get reply
$client->send("HELLO");
$reply = $client->recv();
printf("I: client status: %s%s", $reply, PHP_EOL);
}
}
function s_worker_socket($context)
{
$worker = new ZMQSocket($context, ZMQ::SOCKET_DEALER);
// Set random identity to make tracing easier
$identity = sprintf("%04X-%04X", rand(0, 0x10000), rand(0, 0x10000));
$worker->setSockOpt(ZMQ::SOCKOPT_IDENTITY, $identity);
$worker->connect("tcp://localhost:5556");
// Configure socket to not wait at close time
$worker->setSockOpt(ZMQ::SOCKOPT_LINGER, 0);
// Tell queue we're ready for work
printf("I: (%s) worker ready%s", $identity, PHP_EOL);
$worker->send("READY");
return array($worker, $identity);
}
function __call($name, $arguments)
{
$context = new \ZMQContext();
$zmq = new \ZMQSocket($context, \ZMQ::SOCKET_REQ);
$zmq->connect($this->socket);
$message = array('jsonrpc' => '2.0', 'method' => $name, 'params' => $arguments, 'id' => round(microtime(true) * 100000));
/** @var \ZMQSocket $resultSocket */
$zmq->send(json_encode($message));
$result = json_decode($zmq->recv(), true);
if (isset($result['error'])) {
throw new RpcException($result['exception']['message'], $result['exception']['code']);
}
return $result['result'];
}
function worker_routine()
{
$context = new ZMQContext();
// Socket to talk to dispatcher
$receiver = new ZMQSocket($context, ZMQ::SOCKET_REP);
$receiver->connect("ipc://workers.ipc");
while (true) {
$string = $receiver->recv();
printf("Received request: [%s]%s", $string, PHP_EOL);
// Do some 'work'
sleep(1);
// Send reply back to client
$receiver->send("World");
}
}
/**
* @param TransactionInterface $transaction
*/
private function processTransaction(TransactionInterface $transaction)
{
$results = [];
$nOut = count($transaction->getOutputs());
for ($i = 0; $i < $nOut; $i++) {
$output = $transaction->getOutput($i);
$script = $output->getScript()->getBinary();
if (!isset($results[$script])) {
$results[$script] = $output->getValue();
} else {
$results[$script] += $output->getValue();
}
}
// Compare results to known contracts
foreach ($this->contracts as $contract) {
$requirements = $contract['requirements'];
$rCount = count($requirements);
$have = 0;
foreach ($requirements as $script => $value) {
if (isset($results[$script])) {
echo 'pmt';
if ($results[$script] >= $value) {
$have++;
}
}
}
if ($have > 0) {
if ($have < $rCount) {
$command = 'tx.partial';
} else {
$command = 'tx.complete';
}
$this->listener->send(json_encode(['slug' => $contract['slug'], 'command' => $command, 'tx' => $transaction->getHex()]));
}
}
}
/**
* Send message to socket. Destroys message after sending.
*
* @throws Exception if no socket present
* @param boolean $clear
* @return Zmsg
*/
public function send($clear = true)
{
if (!isset($this->_socket)) {
throw new Exception("No socket supplied");
}
$count = count($this->_parts);
$i = 1;
foreach ($this->_parts as $part) {
$mode = $i++ == $count ? null : ZMQ::MODE_SNDMORE;
$this->_socket->send($part, $mode);
}
if ($clear) {
unset($this->_parts);
$this->_parts = array();
}
return $this;
}
protected function declareReplyToPm()
{
$this->replyToPmSocket = $this->context->getSocket(\ZMQ::SOCKET_REP);
$this->replyToPmSocket->bind($this->loadManagerDto->getPmLmSocketsParams()->getPmLmRequestAddress());
$this->replyToPmSocket->on(EventsConstants::ERROR, function (\Exception $e) {
$this->logger->error(LoggingExceptions::getExceptionString($e));
});
$this->replyToPmSocket->on(EventsConstants::MESSAGE, function ($receivedDtoContainer) {
/**
* @var DtoContainer $dtoContainer
*/
$dtoContainer = unserialize($receivedDtoContainer);
$this->processReceivedControlDto($dtoContainer->getDto());
$this->receivePmInfo = true;
$this->replyToPmSocket->send(serialize($this->dtoContainer));
});
return null;
}
/**
* Starts an endless running service
*/
public function start()
{
$context = new \ZMQContext();
$zmq = new \ZMQSocket($context, \ZMQ::SOCKET_REP);
$zmq->bind($this->socket);
while (true) {
try {
$message = '' . $zmq->recv();
$object = json_decode($message, true);
$method = $object['method'];
$args = $object['params'];
$id = $object['id'];
$response = $this->handleRequest($id, $method, $args);
$zmq->send(json_encode($response));
} catch (\Exception $e) {
}
}
}
/**
*
*/
public function run()
{
$batchItemCount = 10;
$uri = 'tcp://127.0.0.1:5556';
$rpcData = array("id" => 1, "method" => 'TestStack.Test.ping', "params" => array());
/* Create new queue object */
$queue = new \ZMQSocket(new \ZMQContext(), \ZMQ::SOCKET_PUSH, "MySock1");
$queue->connect($uri);
$rpcBatch = array();
for ($i = 0; $i < $batchItemCount; $i++) {
$rpcData = (array) $rpcData;
$rpcData['id'] = $i;
$rpcBatch[] = $rpcData;
}
$mqData = $rpcBatch;
/* Assign socket 1 to the queue, send and receive */
$queue->send(json_encode($mqData), \ZMQ::MODE_NOBLOCK);
}
function step2()
{
$pid = pcntl_fork();
if ($pid == 0) {
step1();
exit;
}
$context = new ZMQContext();
// Bind to ipc: endpoint, then start upstream thread
$receiver = new ZMQSocket($context, ZMQ::SOCKET_PAIR);
$receiver->bind("ipc://step2.ipc");
// Wait for signal
$receiver->recv();
// Signal downstream to step 3
$sender = new ZMQSocket($context, ZMQ::SOCKET_PAIR);
$sender->connect("ipc://step3.ipc");
$sender->send("");
}
function worker_thread()
{
$context = new ZMQContext();
$worker = new ZMQSocket($context, ZMQ::SOCKET_REQ);
$worker->connect("ipc://routing.ipc");
$total = 0;
while (true) {
// Tell the router we're ready for work
$worker->send("ready");
// Get workload from router, until finished
$workload = $worker->recv();
if ($workload == 'END') {
printf("Processed: %d tasks%s", $total, PHP_EOL);
break;
}
$total++;
// Do some random work
usleep(mt_rand(1, 1000000));
}
}
/**
* @return null
*/
protected function finishIteration()
{
$this->iAmSubscriber = 0;
$this->considerMeAsSubscriber = 0;
$this->doNotConsiderMeAsSubscriber = 0;
$this->replyStackReturnResult = false;
$this->startAwaitBeReadyToAct = microtime(true);
$this->getRequestForStartFromReplyStack = false;
$this->publishWasMade = false;
$this->resultingPushMessages = [];
//$this->performerImitatorActive = false;
$this->performerImitationRequests = 0;
if ($this->sleepForPeriod > 0) {
$this->logger->info("Start sleep for (microseconds) " . $this->sleepForPeriod);
usleep($this->sleepForPeriod);
$this->logger->info("Finish sleep for (microseconds) " . $this->sleepForPeriod);
$this->sleepForPeriod = 0;
}
$this->replyToReplyStack->send(serialize(new PulsarIterationFinish()));
$this->logger->debug("Pulsar finish whole iteration with timer iteration number " . $this->timerIterationsNumber . " and set false/zero values to relevant properties.");
return null;
}
