public function execute()
{
global $wgPoolCounterConf, $wgCirrusSearchMaintenanceTimeout;
$wiki = sprintf("[%20s]", wfWikiId());
// Set the timeout for maintenance actions
$this->getConnection()->setTimeout($wgCirrusSearchMaintenanceTimeout);
// Make sure we've actually got indicies to populate
if (!$this->simpleCheckIndexes()) {
$this->error("{$wiki} index(es) do not exist. Did you forget to run updateSearchIndexConfig?", 1);
}
// Make sure we don't flood the pool counter
unset($wgPoolCounterConf['CirrusSearch-Search']);
if (!is_null($this->getOption('from')) || !is_null($this->getOption('to'))) {
// 0 is falsy so MWTimestamp makes that `now`. '00' is epoch 0.
$this->fromDate = new MWTimestamp($this->getOption('from', '00'));
$this->toDate = new MWTimestamp($this->getOption('to', false));
}
$this->toId = $this->getOption('toId');
$this->indexUpdates = !$this->getOption('deletes', false);
$this->limit = $this->getOption('limit');
$buildChunks = $this->getOption('buildChunks');
if ($buildChunks !== null) {
$this->buildChunks($buildChunks);
return;
}
$this->queue = $this->getOption('queue');
$this->maxJobs = $this->getOption('maxJobs') ? intval($this->getOption('maxJobs')) : null;
$this->pauseForJobs = $this->getOption('pauseForJobs') ? intval($this->getOption('pauseForJobs')) : $this->maxJobs;
$updateFlags = 0;
if ($this->getOption('indexOnSkip')) {
$updateFlags |= Updater::INDEX_ON_SKIP;
}
if ($this->getOption('skipParse')) {
$updateFlags |= Updater::SKIP_PARSE;
if (!$this->getOption('batch-size')) {
$this->setBatchSize(50);
}
}
if ($this->getOption('skipLinks')) {
$updateFlags |= Updater::SKIP_LINKS;
}
$this->namespace = $this->hasOption('namespace') ? intval($this->getOption('namespace')) : null;
if ($this->indexUpdates) {
if ($this->queue) {
$operationName = 'Queued';
} else {
$operationName = 'Indexed';
}
} else {
$operationName = 'Deleted';
}
$operationStartTime = microtime(true);
$lastJobQueueCheckTime = 0;
$completed = 0;
$rate = 0;
$minUpdate = $this->fromDate;
if ($this->indexUpdates) {
$minId = $this->getOption('fromId', -1);
} else {
$minNamespace = -100000000;
$minTitle = '';
}
while (is_null($this->limit) || $this->limit > $completed) {
if ($this->indexUpdates) {
$updates = $this->findUpdates($minUpdate, $minId, $this->toDate);
$size = count($updates);
// Note that we'll strip invalid updates after checking to the loop break condition
// because we don't want a batch the contains only invalid updates to cause early
// termination of the process....
} else {
$deletes = $this->findDeletes($minUpdate, $minNamespace, $minTitle, $this->toDate);
$size = count($deletes);
}
if ($size == 0) {
break;
}
if ($this->indexUpdates) {
$last = $updates[$size - 1];
// We make sure to set this if we need it but don't bother when we don't because
// it requires loading the revision.
if (isset($last['update'])) {
$minUpdate = $last['update'];
}
$minId = $last['id'];
// Strip updates down to just pages
$pages = array();
foreach ($updates as $update) {
if (isset($update['page'])) {
$pages[] = $update['page'];
}
}
if ($this->queue) {
$now = microtime(true);
if ($now - $lastJobQueueCheckTime > self::SECONDS_BETWEEN_JOB_QUEUE_LENGTH_CHECKS) {
$lastJobQueueCheckTime = $now;
$queueSize = $this->getUpdatesInQueue();
if ($this->maxJobs !== null && $this->maxJobs < $queueSize) {
do {
$this->output("{$wiki} Waiting while job queue shrinks: {$this->pauseForJobs} > {$queueSize}\n");
usleep(self::SECONDS_BETWEEN_JOB_QUEUE_LENGTH_CHECKS * 1000000);
$queueSize = $this->getUpdatesInQueue();
} while ($this->pauseForJobs < $queueSize);
}
}
JobQueueGroup::singleton()->push(Job\MassIndex::build($pages, $updateFlags, $this->getOption('cluster')));
} else {
// Update size with the actual number of updated documents.
$updater = $this->createUpdater();
$size = $updater->updatePages($pages, null, null, $updateFlags);
}
} else {
$titlesToDelete = array();
$idsToDelete = array();
foreach ($deletes as $delete) {
$titlesToDelete[] = $delete['title'];
$idsToDelete[] = $delete['page'];
$lastDelete = $delete;
}
$minUpdate = $lastDelete['timestamp'];
$minNamespace = $lastDelete['title']->getNamespace();
$minTitle = $lastDelete['title']->getText();
$updater = $this->createUpdater();
$updater->deletePages($titlesToDelete, $idsToDelete);
}
$completed += $size;
$rate = round($completed / (microtime(true) - $operationStartTime));
if (is_null($this->toDate)) {
$endingAt = $minId;
} else {
$endingAt = $minUpdate->getTimestamp(TS_ISO_8601);
}
$this->output("{$wiki} {$operationName} {$size} pages ending at {$endingAt} at {$rate}/second\n");
}
$this->output("{$operationName} a total of {$completed} pages at {$rate}/second\n");
$lastQueueSizeForOurJob = PHP_INT_MAX;
$waitStartTime = microtime(true);
if ($this->queue) {
$this->output("Waiting for jobs to drain from the queue\n");
while (true) {
$queueSizeForOurJob = $this->getUpdatesInQueue();
if ($queueSizeForOurJob === 0) {
break;
}
// We subtract 5 because we some jobs may be added by deletes
if ($queueSizeForOurJob > $lastQueueSizeForOurJob) {
$this->output("Queue size went up. Another script is likely adding jobs " . "and it'll wait for them to empty.\n");
break;
}
if (microtime(true) - $waitStartTime > 120) {
// Wait at least two full minutes before we check if the job count went down.
// Less then that and we might be seeing lag from redis's counts.
$lastQueueSizeForOurJob = $queueSizeForOurJob;
}
$this->output("{$wiki} {$queueSizeForOurJob} jobs left on the queue.\n");
usleep(self::SECONDS_BETWEEN_JOB_QUEUE_LENGTH_CHECKS * 1000000);
}
}
}
/**
* Called when a page is imported. Force a full index of the page. Use the MassIndex
* job since there's likely to be a bunch and we'll prioritize them well but use
* INDEX_EVERYTHING since we won't get a chance at a second pass.
*
* @param Title $title The page title we've just imported
* @return bool
*/
public static function onAfterImportPage($title)
{
// The title can be null if the import failed. Nothing to do in that case.
if ($title === null) {
return;
}
JobQueueGroup::singleton()->push(Job\MassIndex::build(array(WikiPage::factory($title)), Updater::INDEX_EVERYTHING));
return true;
}
/**
* @dataProvider workItemCountTestCases
*/
public function testWorkItemCount($pageDBKeys, $expected)
{
$job = new MassIndex(Title::newMainPage(), array('pageDBKeys' => $pageDBKeys));
$this->assertEquals($expected, $job->workItemCount());
}
