function client_thread($self)
{
$context = new ZMQContext();
$client = new ZMQSocket($context, ZMQ::SOCKET_REQ);
$endpoint = sprintf("ipc://%s-localfe.ipc", $self);
$client->connect($endpoint);
$monitor = new ZMQSocket($context, ZMQ::SOCKET_PUSH);
$endpoint = sprintf("ipc://%s-monitor.ipc", $self);
$monitor->connect($endpoint);
$readable = $writeable = array();
while (true) {
sleep(mt_rand(0, 4));
$burst = mt_rand(1, 14);
while ($burst--) {
// Send request with random hex ID
$task_id = sprintf("%04X", mt_rand(0, 10000));
$client->send($task_id);
// Wait max ten seconds for a reply, then complain
$poll = new ZMQPoll();
$poll->add($client, ZMQ::POLL_IN);
$events = $poll->poll($readable, $writeable, 10 * 1000000);
if ($events > 0) {
foreach ($readable as $socket) {
$zmsg = new Zmsg($socket);
$zmsg->recv();
// Worker is supposed to answer us with our task id
assert($zmsg->body() == $task_id);
}
} else {
$monitor->send(sprintf("E: CLIENT EXIT - lost task %s", $task_id));
exit;
}
}
}
}
public function __construct(CoffeeService $service)
{
$this->service = $service;
$context = new ZMQContext();
$this->socket = $context->getSocket(\ZMQ::SOCKET_PUSH, 'pusher');
$this->socket->connect('tcp://127.0.0.1:4444');
}
/**
* @param string $request
*/
public function request($request)
{
// Prefix request with sequence number and empty envelope
$this->sequence++;
$msg = array('', $this->sequence, $request);
// Blast the request to all connected servers
for ($server = 1; $server <= $this->servers; $server++) {
$this->socket->sendMulti($msg);
}
// Wait for a matching reply to arrive from anywhere
// Since we can poll several times, calculate each one
$poll = new ZMQPoll();
$poll->add($this->socket, ZMQ::POLL_IN);
$reply = null;
$endtime = time() + self::GLOBAL_TIMEOUT / 1000;
while (time() < $endtime) {
$readable = $writable = array();
$events = $poll->poll($readable, $writable, ($endtime - time()) * 1000);
foreach ($readable as $sock) {
if ($sock == $this->socket) {
$reply = $this->socket->recvMulti();
if (count($reply) != 3) {
exit;
}
$sequence = $reply[1];
if ($sequence == $this->sequence) {
break;
}
}
}
}
return $reply;
}
/**
* {@inheritDoc}
*/
public function receiveResponse()
{
try {
$result = $this->socket->recvMulti();
} catch (\ZMQSocketException $e) {
throw new TransportException('Cannot receive response', 0, $e);
}
if ($result === false) {
throw new TimeoutException('Timeout (' . $this->getTimeout() . 's) reached');
}
if (!isset($result[0])) {
throw new FormatException('Invalid response - no response type', $result);
}
if (!isset($result[1])) {
throw new FormatException('Invalid response - no headers', $result);
}
$response = new Response($result[0]);
$response->setHeaders(new Headers(Parser::parseHeaders($result[1])));
if (!MessageTypes::isResponseTypeWithResult($response->getType())) {
return $response;
}
if (!isset($result[2])) {
throw new FormatException('Invalid response - no response body', $result);
}
return $response->setResultBody($result[2]);
}
public function run(OutputInterface $output)
{
$context = new \ZMQContext();
$tasksQueue = new \ZMQSocket($context, \ZMQ::SOCKET_PULL);
$tasksQueue->connect(Spider::ZMQ_TASKS_QUEUE_DSN);
$resultsQueue = new \ZMQSocket($context, \ZMQ::SOCKET_PUSH);
$resultsQueue->connect(Spider::ZMQ_RESULTS_QUEUE_DSN);
$output->writeln('HTTP Worker is waiting for tasks');
while (true) {
$string = $tasksQueue->recv();
if ($string === Spider::ZMQ_COMMAND_BATCH_START) {
$resultsQueue->send($string);
} elseif (stripos($string, Spider::ZMQ_COMMAND_BATCH_END) !== false) {
// send info for result collector how many results it should expect
$resultsQueue->send($string);
} elseif ($string === Spider::ZMQ_COMMAND_WORKER_QUIT) {
$output->writeln('No more work. Worker stops now.');
break;
// no more work
} else {
$output->writeln('Fetching data from URI: ' . $string);
$userData = file_get_contents($string);
// TODO: use Guzzle
$output->writeln('Sending result for URI: ' . $string);
$resultsQueue->send($userData);
}
}
}
/**
* Notifies the task manager given a message constant, see MESSAGE_* constants.
*
* @param string $message
*
* @return mixed|null The return value of the task manager.
*
* @throws RuntimeException in case notification did not occur within the timeout.
*/
public function notify($message)
{
try {
$command = $this->createCommand($message);
$this->socket->send($command);
$result = false;
$limit = microtime(true) + $this->timeout;
while (microtime(true) < $limit && false === ($result = $this->socket->recv(\ZMQ::MODE_NOBLOCK))) {
usleep(1000);
}
if (false === $result) {
$this->logger->error(sprintf('Unable to notify the task manager with message "%s" within timeout of %d seconds', $message, $this->timeout));
throw new RuntimeException('Unable to retrieve information.');
}
$data = @json_decode($result, true);
if (JSON_ERROR_NONE !== json_last_error()) {
throw new RuntimeException('Invalid task manager response : invalid JSON.');
}
if (!isset($data['reply']) || !isset($data['request']) || $command !== $data['request']) {
throw new RuntimeException('Invalid task manager response : missing fields.');
}
return $data['reply'];
} catch (\ZMQSocketException $e) {
$this->logger->error(sprintf('Unable to notify the task manager with message "%s" within timeout of %d seconds', $message, $this->timeout), ['exception' => $e]);
throw new RuntimeException('Unable to retrieve information.', $e->getCode(), $e);
}
}
function broker_task()
{
// Prepare our context and sockets
$context = new ZMQContext();
$frontend = new ZMQSocket($context, ZMQ::SOCKET_ROUTER);
$backend = new ZMQSocket($context, ZMQ::SOCKET_ROUTER);
$frontend->bind("tcp://*:5555");
$backend->bind("tcp://*:5556");
// Initialize poll set
$poll = new ZMQPoll();
$poll->add($frontend, ZMQ::POLL_IN);
$poll->add($backend, ZMQ::POLL_IN);
$read = $write = array();
while (true) {
$events = $poll->poll($read, $write);
foreach ($read as $socket) {
$zmsg = new Zmsg($socket);
$zmsg->recv();
if ($socket === $frontend) {
$zmsg->push("W");
$zmsg->set_socket($backend)->send();
} elseif ($socket === $backend) {
$zmsg->pop();
$zmsg->push("C");
$zmsg->set_socket($frontend)->send();
}
}
}
}
/**
* @param string $message
* @return mixed
*/
public function send($message)
{
if ($this->isVerbose()) {
$this->logger->debug("Sending message: " . $message);
}
$this->socket->send($message);
}
/**
* Start the worker and wait for requests
*/
public function listen()
{
$context = new \ZMQContext();
$server = new \ZMQSocket($context, \ZMQ::SOCKET_PULL);
$server->bind('tcp://127.0.0.1:' . ($this->defaultPort + $this->client->getId() - 1));
$this->logger->info('Client worker ' . $this->client . ' is ready');
while (true) {
$request = $server->recv();
$this->logger->debug('Client worker ' . $this->client . ' receiving request : ' . $request);
// Check if the input is valid, ignore if wrong
$request = json_decode($request, true);
if (!$this->isValidInput($request)) {
$this->logger->error('Client worker ' . $this->client . ' received an invalid input');
continue;
}
try {
// Call the right method in the client and push to redis the result
$result = call_user_func_array(array($this->client, $request['command']), $request['parameters']);
} catch (ClientNotReadyException $e) {
$this->logger->warning('Client worker ' . $this->client . ' received a request (#' . $request['invokeId'] . ') whereas the client is not ready. This is normal in case of client reconnection process. Ignoring.');
continue;
}
$key = $this->key . '.client.commands.' . $request['invokeId'];
$this->redis->rpush($key, serialize($result));
$this->redis->expire($key, $this->expire);
}
}
/**
* @param $context
* @param $endpoint
* @return Zmsg
*/
private function createPublisher($context, $endpoint)
{
$publisher = new \ZMQSocket($context, \ZMQ::SOCKET_PUB);
$publisher->setSockOpt(\ZMQ::SOCKOPT_SNDHWM, 1);
$publisher->connect($endpoint);
return new Zmsg($publisher);
}
/**
* {@inheritdoc}
*/
public function close()
{
if ($this->socket instanceof \ZMQSocket && !$this->socket->isPersistent()) {
$this->socket->disconnect($this->dsn);
}
$this->socket = null;
}
/**
* Connect to ZMQ and set socket
*
* @return ZMQ $socket instance
*/
protected function connectToZMQ()
{
$context = \App::make('ZMQContext');
$this->socket = $context->getSocket(\ZMQ::SOCKET_PUSH, \Config::get('larapush::persistent_socket_name'));
$this->socket->connect($this->getPusherConnect());
return $this->socket;
}
public function collect(OutputInterface $output)
{
$context = new \ZMQContext();
$resultsQueue = new \ZMQSocket($context, \ZMQ::SOCKET_PULL);
$resultsQueue->bind(Spider::ZMQ_RESULTS_QUEUE_BIND_DSN);
$statusQueue = new \ZMQSocket($context, \ZMQ::SOCKET_PUSH);
$statusQueue->bind(Spider::ZMQ_STATUS_QUEUE_BIND_DSN);
$tstart = microtime(true);
$collectedResults = 0;
$expectedResults = PHP_INT_MAX;
$output->writeln('Collecting Task results');
while ($collectedResults < $expectedResults) {
$string = $resultsQueue->recv();
if ($string === Spider::ZMQ_COMMAND_BATCH_START) {
// Wait for start of batch
} elseif (stripos($string, Spider::ZMQ_COMMAND_BATCH_END) === false) {
$output->writeln('Got task result: ' . substr($string, 0, 20) . ' ...');
file_put_contents($this->resultsTargetPath . '/' . md5($string) . '.result', $string);
// TODO: use Symfony/Filesystem
$output->writeln('Collected results so far: ' . ++$collectedResults);
} else {
$expectedResults = (int) explode('%', $string)[1];
$output->writeln('[INFO] Trying to collect ' . $expectedResults . ' as requested by Task Loader');
}
}
$tend = microtime(true);
$totalMsec = ($tend - $tstart) * 1000;
$output->writeln('Task results collecting finished. Got ' . $collectedResults . ' results');
$output->writeln("Total elapsed time: {$totalMsec} msec");
$output->writeln('Sending Task Result Collector info');
$statusQueue->send($collectedResults);
}
public function testMissingRequestReturnNull()
{
$this->socket->expects($this->once())->method('send')->with(TaskManager::MESSAGE_PROCESS_UPDATE);
$this->socket->expects($this->once())->method('recv')->will($this->returnValue(json_encode(['request' => TaskManager::MESSAGE_PROCESS_UPDATE])));
$this->setExpectedException(RuntimeException::class, 'Invalid task manager response : missing fields.');
$this->sut->notify(Notifier::MESSAGE_CREATE);
}
/**
* Run ZMQ interface for generator
*
* Req-rep pattern; msgs are commands:
*
* GEN = Generate ID
* STATUS = Get status string
*/
public function run()
{
$context = new \ZMQContext();
$receiver = new \ZMQSocket($context, \ZMQ::SOCKET_REP);
$bindTo = 'tcp://*:' . $this->port;
echo "Binding to {$bindTo}\n";
$receiver->bind($bindTo);
while (TRUE) {
$msg = $receiver->recv();
switch ($msg) {
case 'GEN':
try {
$response = $this->generator->generate();
} catch (\Exception $e) {
$response = "ERROR";
}
break;
case 'STATUS':
$response = json_encode($this->generator->status());
break;
default:
$response = 'UNKNOWN COMMAND';
break;
}
$receiver->send($response);
}
}
public function connect()
{
$context = new \ZMQContext();
$socket = new \ZMQSocket($context, \ZMQ::SOCKET_SUB);
$socket->bind($this->dsn());
return $socket;
}
/**
* @return \ZMQSocket
*/
private function getSocket()
{
if (null == $this->pushSocket) {
$this->pushSocket = $this->context->getSocket(\ZMQ::SOCKET_PUSH, 'my pusher');
$this->pushSocket->connect("tcp://localhost:5555");
}
return $this->pushSocket;
}
/**
* @param $context
* @param $endpoint
* @return Zmsg
*/
private function createCollector($context, $endpoint)
{
$receiver = new \ZMQSocket($context, \ZMQ::SOCKET_SUB);
$receiver->setSockOpt(\ZMQ::SOCKOPT_LINGER, 0);
$receiver->setSockOpt(\ZMQ::SOCKOPT_SUBSCRIBE, "");
$receiver->bind($endpoint);
return $receiver;
}
/**
* @test
*/
public function it_sends_message_and_gets_response()
{
$socket = new \ZMQSocket(new \ZMQContext(), \ZMQ::SOCKET_REQ);
$zmqClient = new ZeroMQSocket($socket, 'tcp://localhost:5556');
$zmqClient->send($message = 'testing-123');
$this->assertEquals($message, $zmqClient->receive());
$socket->disconnect('tcp://localhost:5556');
}
/**
* @param $context
* @param $endpoint
* @return Zmsg
*/
private function createSubscriber($context, $endpoint)
{
$receiver = new \ZMQSocket($context, \ZMQ::SOCKET_SUB);
$receiver->setSockOpt(\ZMQ::SOCKOPT_LINGER, 0);
$receiver->setSockOpt(\ZMQ::SOCKOPT_SUBSCRIBE, "");
$receiver->connect($endpoint);
return new Zmsg($receiver);
}
/**
* @param RawZMQSocket $socket
* @return ZmqSocket
*/
private function wrapSocket(RawZMQSocket $socket)
{
$wrapped = new ZmqSocket($socket, $this->loop);
if ($this->isReadableSocketType($socket->getSocketType())) {
$wrapped->attachReadListener();
}
return $wrapped;
}
/**
* {@inheritDoc}
*/
protected function write(array $record)
{
if ($this->multipart) {
$this->zmqSocket->send($record['channel'], $this->zmqMode);
$this->zmqSocket->send($record['formatted']);
} else {
$this->zmqSocket->send($record["formatted"], $this->zmqMode);
}
}
/**
* @return \ZMQSocket
*/
public function connect()
{
$context = new \ZMQContext();
$socket = new \ZMQSocket($context, \ZMQ::SOCKET_PUB);
$socket->connect($this->dsn());
// @need some sleep at :-(
usleep(500);
return $socket;
}
function client_socket(ZMQContext $context)
{
echo "I: connecting to server...", PHP_EOL;
$client = new ZMQSocket($context, ZMQ::SOCKET_REQ);
$client->connect("tcp://localhost:5555");
// Configure socket to not wait at close time
$client->setSockOpt(ZMQ::SOCKOPT_LINGER, 0);
return $client;
}
/**
* @param string|array $message
* @param int $mode
*/
public function send($message, $mode)
{
if (false === $this->connected) {
$connectedTo = $this->socket->getEndpoints();
if (!in_array($this->dsn, $connectedTo)) {
$this->socket->connect($this->dsn);
}
$this->connected = true;
}
$this->socket->send($message, $mode);
}
function url_hash_worker()
{
// Socket to talk to dispatcher
$context = new ZMQContext();
$receiver = new ZMQSocket($context, ZMQ::SOCKET_REP);
$receiver->connect("ipc://urlhash.ipc");
while (true) {
$hashedUrl = $receiver->recv();
$receiver->send(hashDecode($hashedUrl));
}
}
public function testComplex()
{
$context = new \ZMQContext();
$output = new \ZMQSocket($context, \ZMQ::SOCKET_DEALER);
$output->bind("inproc://zmsg_selftest");
$input = new \ZMQSocket($context, \ZMQ::SOCKET_ROUTER);
$input->connect("inproc://zmsg_selftest");
// Test send and receive of single-part message
$zmsgo = new Zmsg($output);
$zmsgo->setLast("Hello");
$this->assertTrue($zmsgo->getLast() == "Hello");
$zmsgo->send();
$zmsgi = new Zmsg($input);
$zmsgi->recv();
$this->assertTrue($zmsgi->parts() == 2);
$this->assertTrue($zmsgi->getLast() == "Hello");
// Test send and receive of multi-part message
$zmsgo = new Zmsg($output);
$zmsgo->setLast("Hello");
$zmsgo->wrap("address1", "");
$zmsgo->wrap("address2");
$this->assertTrue($zmsgo->parts() == 4);
$zmsgo->send();
$zmsgi = new Zmsg($input);
$zmsgi->recv();
$this->assertTrue($zmsgi->parts() == 5);
$zmsgi->unwrap();
$this->assertTrue($zmsgi->unwrap() == "address2");
$zmsgi->setLast(sprintf("%s%s", 'W', "orld"));
$this->assertTrue($zmsgi->getLast() == "World");
// Pull off address 1, check that empty part was dropped
$zmsgi->unwrap();
$this->assertTrue($zmsgi->parts() == 1);
// Check that message body was correctly modified
$part = $zmsgi->pop();
$this->assertTrue($part == "World");
$this->assertTrue($zmsgi->parts() == 0);
// Test load and save
$zmsg = new Zmsg();
$zmsg->setLast("Hello");
$zmsg->wrap("address1", "");
$zmsg->wrap("address2");
$this->assertTrue($zmsg->parts() == 4);
$fh = fopen(sys_get_temp_dir() . "/zmsgtest.zmsg", 'w');
$zmsg->save($fh);
fclose($fh);
$fh = fopen(sys_get_temp_dir() . "/zmsgtest.zmsg", 'r');
$zmsg2 = new Zmsg();
$zmsg2->load($fh);
assert($zmsg2->getLast() == $zmsg->getLast());
fclose($fh);
$this->assertTrue($zmsg2->parts() == 4);
}
/**
* Connects or reconnect to broker
*
* @return AsynClient
*/
public function connect()
{
if ($this->client) {
unset($this->client);
}
$this->client = new ZMQSocket($this->context, ZMQ::SOCKET_DEALER);
$this->client->setSockOpt(ZMQ::SOCKOPT_LINGER, 0);
$this->client->connect($this->broker);
if ($this->verbose) {
$this->log("ZMQDEBUG", "connecting to broker at %s...", $this->broker);
}
return $this;
}
function publisher()
{
$context = new ZMQContext();
// Prepare publisher
$publisher = new ZMQSocket($context, ZMQ::SOCKET_PUB);
$publisher->bind("tcp://*:5556");
while (true) {
// Send current clock (msecs) to subscribers
$publisher->send(microtime(true));
usleep(1000);
// 1msec wait
}
}
/**
* 构造函数
*
* @params $context ZMQContext
* @params $endpoints 套接字链接到的端点
*/
public function __construct($context, $endpoints)
{
if (APS::get_instance()->get_zmq_enabled()) {
$socket = new ZMQSocket($context, ZMQ::SOCKET_XREQ);
$socket->setsockopt(ZMQ::SOCKOPT_LINGER, 0);
$socket->setsockopt(ZMQ::SOCKOPT_HWM, 1000);
foreach ($endpoints as $endpoint) {
$socket->connect($endpoint);
}
self::$sockets[] = $socket;
$this->socket = $socket;
}
}
