public function processQueue()
{
$queue = Database::queue();
$sites = Database::sites();
$c = $queue->find();
$c->sort(array('rnd' => 1));
$c->limit(20);
$cnt = 0;
while ($item = $c->getNext()) {
$cnt++;
$site = $sites->findOne(array('code' => $item['site']));
if (null === $site) {
continue;
}
$parser = $this->initParser($site);
if ('job' === $item['type']) {
$time = time();
if ($parser->processJob($item['id'], $item['url'])) {
$queue->remove(array('_id' => $item['_id']));
// stamps must be unique
while (time() == $time) {
sleep(1);
}
}
}
}
if ($cnt > 0) {
$this->updateGlobalRSS();
}
}
protected function queueJobLink($jobId, $link)
{
$info = array('site' => $this->getSiteCode(), 'id' => (string) $jobId);
$tmp = Database::jobs()->findOne($info);
if (null != $tmp) {
return false;
}
$info['type'] = 'job';
$tmp = Database::queue()->findOne($info);
if (null != $tmp) {
return false;
}
echo '[' . date('H:m:s') . '] Queueing job [' . $link . "]\n";
$info['url'] = $link;
$info['rnd'] = rand(1, 10000);
Database::queue()->insert($info);
$this->queuedCount++;
return true;
}
