public function execute()
{
global $wgFlowSearchMaintenanceTimeout;
// Set the timeout for maintenance actions
Connection::getSingleton()->setTimeout2($wgFlowSearchMaintenanceTimeout);
/** @var Updater[] $updaters */
$updaters = Container::get('searchindex.updaters');
foreach ($updaters as $updaterType => $updater) {
$fromId = $this->getOption('fromId', null);
$fromId = $fromId ? UUID::create($fromId) : null;
$toId = $this->getOption('toId', null);
$toId = $toId ? UUID::create($toId) : null;
$namespace = $this->getOption('namespace', null);
$numRevisionsToIndex = $this->getOption('limit', null);
$total = 0;
while (true) {
// if a limit was provided, we should make sure to not fetch
// more revisions than asked for
$options = array('LIMIT' => $this->mBatchSize);
if ($numRevisionsToIndex) {
$options['LIMIT'] = min($numRevisionsToIndex, $this->mBatchSize);
// since we do this in batches, we'll subtract the size of
// each batch until $numRevisionsToIndex is reached
$numRevisionsToIndex -= $this->mBatchSize;
if ($options['LIMIT'] <= 0) {
break;
}
}
$conditions = $updater->buildQueryConditions($fromId, $toId, $namespace);
$revisions = $updater->getRevisions($conditions, $options);
// stop if we're all out of revisions
if (!$revisions) {
break;
}
$total += $updater->updateRevisions($revisions, null, null);
$this->output("Indexed {$total} {$updaterType} document(s)\n");
// prepare for next batch, starting at the next id
// prevFromId will default to around unix epoch - there can be
// no data before that
$prevFromId = $fromId ?: UUID::getComparisonUUID('1');
$fromId = $this->getNextFromId($revisions);
// make sure we don't get stuck in an infinite loop
$diff = $prevFromId->getTimestampObj()->diff($fromId->getTimestampObj());
// invert will be 1 if the diff is a negative time period from
// $prevFromId to $fromId, which means that the new $timestamp is
// more recent than our current $result
if ($diff->invert) {
$this->error('Got stuck in an infinite loop.' . "\n" . 'workflow_last_update_timestamp is likely incorrect ' . 'for some workflows.' . "\n" . 'Run maintenance/FlowFixWorkflowLastUpdateTimestamp.php ' . 'to automatically fix those.', 1);
}
// prevent memory from being filled up
Container::get('storage')->clear();
}
}
}
/**
* @param \Elastica\Document[] $documents
* @param string|null $shardTimeout Timeout in Elasticsearch time format (1m, 15s, ...)
*/
protected function sendDocuments(array $documents, $shardTimeout = null)
{
if (count($documents) === 0) {
return;
}
try {
// addDocuments (notice plural) is the bulk api
$bulk = new \Elastica\Bulk(Connection::getSingleton()->getClient2());
if ($shardTimeout !== null) {
$bulk->setShardTimeout($shardTimeout);
}
$index = Connection::getFlowIndex(wfWikiId());
$type = $index->getType($this->getTypeName());
$bulk->setType($type);
$bulk->addDocuments($documents);
$bulk->send();
} catch (\Exception $e) {
$documentIds = array_map(function ($doc) {
return $doc->getId();
}, $documents);
wfWarn(__METHOD__ . ': Failed updating documents (' . implode(',', $documentIds) . '): ' . $e->getMessage());
}
}
/**
* @return Validator[]
*/
protected function getValidators()
{
$validators = array();
if ($this->getOption('justCacheWarmers', false)) {
$validators[] = new CacheWarmersValidator($this->indexType, $this->getTopicType(), $this->cacheWarmers, $this);
$validators[] = new CacheWarmersValidator($this->indexType, $this->getHeaderType(), $this->cacheWarmers, $this);
return $validators;
}
if ($this->getOption('justAllocation', false)) {
$validators[] = new ShardAllocationValidator($this->getIndex(), $this->indexAllocation, $this);
return $validators;
}
$validators[] = new IndexValidator($this->getIndex(), $this->startOver, $this->maxShardsPerNode, $this->getShardCount(), $this->getReplicaCount(), $this->refreshInterval, false, $this->analysisConfigBuilder, $this->getMergeSettings(), $this);
$validators = array_merge($validators, $this->getIndexSettingsValidators());
$validator = new AnalyzersValidator($this->getIndex(), $this->analysisConfigBuilder, $this);
$validator->printDebugCheckConfig($this->printDebugCheckConfig);
$validators[] = $validator;
$types = array('topic' => $this->getTopicType(), 'header' => $this->getHeaderType());
$validator = new MappingValidator($this->getIndex(), $this->optimizeIndexForExperimentalHighlighter, $this->availablePlugins, $this->getMappingConfig(), $types, $this);
$validator->printDebugCheckConfig($this->printDebugCheckConfig);
$validators[] = $validator;
$validators[] = new CacheWarmersValidator($this->indexType, $this->getTopicType(), $this->cacheWarmers, $this);
$validators[] = new CacheWarmersValidator($this->indexType, $this->getHeaderType(), $this->cacheWarmers, $this);
$types = array($this->getTopicType(), $this->getHeaderType());
$oldTypes = array($this->getOldTopicType(), $this->getOldHeaderType());
$reindexer = new Reindexer($this->getIndex(), Connection::getSingleton(), $types, $oldTypes, $this->getShardCount(), $this->getReplicaCount(), $this->maintenanceTimeout, $this->getMergeSettings(), $this->getMappingConfig(), $this);
$reindexParams = array($this->reindexProcesses, $this->refreshInterval, $this->reindexRetryAttempts, $this->reindexChunkSize, $this->reindexAcceptableCountDeviation);
$reindexValidators = $this->getIndexSettingsValidators();
$validators[] = new SpecificAliasValidator($this->getClient(), $this->getIndexTypeName(), $this->getSpecificIndexName(), $this->startOver, $reindexer, $reindexParams, $reindexValidators, $this->reindexAndRemoveOk, $this->tooFewReplicas, $this);
$validators[] = new IndexAllAliasValidator($this->getClient(), $this->getIndexName(), $this->getSpecificIndexName(), $this->startOver, $this->getIndexTypeName(), $this);
if ($this->tooFewReplicas) {
$validators = array_merge($validators, $this->getIndexSettingsValidators());
}
return $validators;
}
