/**
* Show the asynchronous client logs in the main process logger (console)
*
* @return string|null
*
* @throws \RuntimeException
*/
public static function subscribe()
{
if (!isset(self::$redisClient)) {
throw new \RuntimeException('Redis client has not been initialised');
}
while (null != ($log = self::$redisClient->lpop(ConfigurationLoader::get('client.async.redis.key') . '.client.logs'))) {
list($level, $message) = explode('|', $log);
self::$logger->addRecord($level, $message);
}
}
/**
* @param string $queue
*/
public function consume($queue)
{
$value = $this->client->lpop($queue);
if ($value === null) {
return;
}
$decoded = json_decode($value, true);
$command = call_user_func($this->resolver, $decoded['command']);
$command->withOptions($decoded['options'])->execute();
}
/**
* Read association from Redis. If no handle given
* and multiple associations found, returns latest issued
*/
function getAssociation($server_url, $handle = null)
{
// simple case: handle given
if ($handle !== null) {
return $this->getAssociationFromServer($this->associationKey($server_url, $handle));
}
// no handle given, receiving the latest issued
$serverKey = $this->associationServerKey($server_url);
$lastKey = $this->redis->lpop($serverKey);
if (!$lastKey) {
return null;
}
// get association, return null if failed
return $this->getAssociationFromServer($lastKey);
}
protected function run($urlMethod, $runMethod)
{
$this->getSize();
// 从redis拿出数据，并定义url，随后开始爬行逻辑
while (static::$count < $this->endCounts) {
if ($len = $this->redis->llen('usernames')) {
for ($i = 0; $i < $len; $i++) {
Crawler::$urlMethod($this->redis->lpop('usernames'));
if (!$this->{$runMethod}()) {
continue;
}
}
static::$count++;
} else {
$this->getUsernames(static::$count * $this->size);
}
}
}
/**
* @inheritdoc
*/
public function pop($queue)
{
foreach ([':delayed', ':reserved'] as $type) {
$options = ['cas' => true, 'watch' => $queue . $type];
$this->redis->transaction($options, function (MultiExec $transaction) use($queue, $type) {
$data = $this->redis->zrangebyscore($queue . $type, '-inf', $time = time());
if (!empty($data)) {
$transaction->zremrangebyscore($queue . $type, '-inf', $time);
$transaction->rpush($queue, $data);
}
});
}
$data = $this->redis->lpop($queue);
if ($data === null) {
return false;
}
$this->redis->zadd($queue . ':reserved', [$data => time() + $this->expire]);
$data = Json::decode($data);
return ['id' => $data['id'], 'body' => $data['body'], 'queue' => $queue];
}
/**
* {@inheritdoc}
*/
public function flushQueue(\Swift_Transport $transport, &$failedRecipients = null)
{
if (!$this->redis->llen($this->key)) {
return 0;
}
if (!$transport->isStarted()) {
$transport->start();
}
$failedRecipients = (array) $failedRecipients;
$count = 0;
$time = time();
while ($message = unserialize($this->redis->lpop($this->key))) {
$count += $transport->send($message, $failedRecipients);
if ($this->getMessageLimit() && $count >= $this->getMessageLimit()) {
break;
}
if ($this->getTimeLimit() && time() - $time >= $this->getTimeLimit()) {
break;
}
}
return $count;
}
/**
* @param int $max
* @return array
*/
public function getCounters($max = 30)
{
$counters = [];
$number = 0;
while (true) {
$redisData = $this->redisClient->lpop($this->getCounterKey());
if (!$redisData) {
break;
}
$counter = unserialize($redisData);
$counters[] = $counter;
$number++;
if ($number >= $max) {
break;
}
}
return $counters;
}