public function execute() { global $wgFlowSearchMaintenanceTimeout; // Set the timeout for maintenance actions Connection::getSingleton()->setTimeout2($wgFlowSearchMaintenanceTimeout); /** @var Updater[] $updaters */ $updaters = Container::get('searchindex.updaters'); foreach ($updaters as $updaterType => $updater) { $fromId = $this->getOption('fromId', null); $fromId = $fromId ? UUID::create($fromId) : null; $toId = $this->getOption('toId', null); $toId = $toId ? UUID::create($toId) : null; $namespace = $this->getOption('namespace', null); $numRevisionsToIndex = $this->getOption('limit', null); $total = 0; while (true) { // if a limit was provided, we should make sure to not fetch // more revisions than asked for $options = array('LIMIT' => $this->mBatchSize); if ($numRevisionsToIndex) { $options['LIMIT'] = min($numRevisionsToIndex, $this->mBatchSize); // since we do this in batches, we'll subtract the size of // each batch until $numRevisionsToIndex is reached $numRevisionsToIndex -= $this->mBatchSize; if ($options['LIMIT'] <= 0) { break; } } $conditions = $updater->buildQueryConditions($fromId, $toId, $namespace); $revisions = $updater->getRevisions($conditions, $options); // stop if we're all out of revisions if (!$revisions) { break; } $total += $updater->updateRevisions($revisions, null, null); $this->output("Indexed {$total} {$updaterType} document(s)\n"); // prepare for next batch, starting at the next id // prevFromId will default to around unix epoch - there can be // no data before that $prevFromId = $fromId ?: UUID::getComparisonUUID('1'); $fromId = $this->getNextFromId($revisions); // make sure we don't get stuck in an infinite loop $diff = $prevFromId->getTimestampObj()->diff($fromId->getTimestampObj()); // invert will be 1 if the diff is a negative time period from // $prevFromId to $fromId, which means that the new $timestamp is // more recent than our current $result if ($diff->invert) { $this->error('Got stuck in an infinite loop.' . "\n" . 'workflow_last_update_timestamp is likely incorrect ' . 'for some workflows.' . "\n" . 'Run maintenance/FlowFixWorkflowLastUpdateTimestamp.php ' . 'to automatically fix those.', 1); } // prevent memory from being filled up Container::get('storage')->clear(); } } }
/** * @param \Elastica\Document[] $documents * @param string|null $shardTimeout Timeout in Elasticsearch time format (1m, 15s, ...) */ protected function sendDocuments(array $documents, $shardTimeout = null) { if (count($documents) === 0) { return; } try { // addDocuments (notice plural) is the bulk api $bulk = new \Elastica\Bulk(Connection::getSingleton()->getClient2()); if ($shardTimeout !== null) { $bulk->setShardTimeout($shardTimeout); } $index = Connection::getFlowIndex(wfWikiId()); $type = $index->getType($this->getTypeName()); $bulk->setType($type); $bulk->addDocuments($documents); $bulk->send(); } catch (\Exception $e) { $documentIds = array_map(function ($doc) { return $doc->getId(); }, $documents); wfWarn(__METHOD__ . ': Failed updating documents (' . implode(',', $documentIds) . '): ' . $e->getMessage()); } }
/** * @return Validator[] */ protected function getValidators() { $validators = array(); if ($this->getOption('justCacheWarmers', false)) { $validators[] = new CacheWarmersValidator($this->indexType, $this->getTopicType(), $this->cacheWarmers, $this); $validators[] = new CacheWarmersValidator($this->indexType, $this->getHeaderType(), $this->cacheWarmers, $this); return $validators; } if ($this->getOption('justAllocation', false)) { $validators[] = new ShardAllocationValidator($this->getIndex(), $this->indexAllocation, $this); return $validators; } $validators[] = new IndexValidator($this->getIndex(), $this->startOver, $this->maxShardsPerNode, $this->getShardCount(), $this->getReplicaCount(), $this->refreshInterval, false, $this->analysisConfigBuilder, $this->getMergeSettings(), $this); $validators = array_merge($validators, $this->getIndexSettingsValidators()); $validator = new AnalyzersValidator($this->getIndex(), $this->analysisConfigBuilder, $this); $validator->printDebugCheckConfig($this->printDebugCheckConfig); $validators[] = $validator; $types = array('topic' => $this->getTopicType(), 'header' => $this->getHeaderType()); $validator = new MappingValidator($this->getIndex(), $this->optimizeIndexForExperimentalHighlighter, $this->availablePlugins, $this->getMappingConfig(), $types, $this); $validator->printDebugCheckConfig($this->printDebugCheckConfig); $validators[] = $validator; $validators[] = new CacheWarmersValidator($this->indexType, $this->getTopicType(), $this->cacheWarmers, $this); $validators[] = new CacheWarmersValidator($this->indexType, $this->getHeaderType(), $this->cacheWarmers, $this); $types = array($this->getTopicType(), $this->getHeaderType()); $oldTypes = array($this->getOldTopicType(), $this->getOldHeaderType()); $reindexer = new Reindexer($this->getIndex(), Connection::getSingleton(), $types, $oldTypes, $this->getShardCount(), $this->getReplicaCount(), $this->maintenanceTimeout, $this->getMergeSettings(), $this->getMappingConfig(), $this); $reindexParams = array($this->reindexProcesses, $this->refreshInterval, $this->reindexRetryAttempts, $this->reindexChunkSize, $this->reindexAcceptableCountDeviation); $reindexValidators = $this->getIndexSettingsValidators(); $validators[] = new SpecificAliasValidator($this->getClient(), $this->getIndexTypeName(), $this->getSpecificIndexName(), $this->startOver, $reindexer, $reindexParams, $reindexValidators, $this->reindexAndRemoveOk, $this->tooFewReplicas, $this); $validators[] = new IndexAllAliasValidator($this->getClient(), $this->getIndexName(), $this->getSpecificIndexName(), $this->startOver, $this->getIndexTypeName(), $this); if ($this->tooFewReplicas) { $validators = array_merge($validators, $this->getIndexSettingsValidators()); } return $validators; }