public function __construct(Connection $conn, array $flags = array()) { parent::__construct($conn, null, null); if (in_array('same-cluster', $flags)) { $this->writeToClusterName = $this->connection->getClusterName(); } }
/** * @return Status */ public function validate() { $this->outputIndented("Validating mappings..."); if ($this->optimizeIndexForExperimentalHighlighter && !in_array('experimental highlighter', $this->availablePlugins)) { $this->output("impossible!\n"); return Status::newFatal(new RawMessage("wgCirrusSearchOptimizeIndexForExperimentalHighlighter is set to true but the " . "'experimental highlighter' plugin is not installed on all hosts.")); } $requiredMappings = $this->mappingConfig; if (!$this->checkMapping($requiredMappings)) { /** @var Mapping[] $actions */ $actions = array(); // TODO Conflict resolution here might leave old portions of mappings foreach ($this->types as $typeName => $type) { $action = new Mapping($type); foreach ($requiredMappings[$typeName] as $key => $value) { $action->setParam($key, $value); } $actions[] = $action; } try { foreach ($actions as $action) { $action->send(); } $this->output("corrected\n"); } catch (ExceptionInterface $e) { $this->output("failed!\n"); $message = ElasticsearchIntermediary::extractMessage($e); return Status::newFatal(new RawMessage("Couldn't update mappings. Here is elasticsearch's error message: {$message}\n")); } } return Status::newGood(); }
/** * @var Connection */ public function __construct(Connection $conn) { parent::__construct($conn, null, null); $this->log = LoggerFactory::getInstance('CirrusSearch'); $this->failedLog = LoggerFactory::getInstance('CirrusSearchChangeFailed'); }
/** * Constructor * @param int $offset Offset the results by this much * @param int $limit Limit the results to this many * @param SearchConfig Configuration settings * @param int[]|null $namespaces Array of namespace numbers to search or null to search all namespaces. * @param User|null $user user for which this search is being performed. Attached to slow request logs. * @param string|boolean $index Base name for index to search from, defaults to wfWikiId() */ public function __construct(Connection $conn, $offset, $limit, SearchConfig $config = null, array $namespaces = null, User $user = null, $index = false) { if (is_null($config)) { // @todo connection has an embeded config ... reuse that? somehow should // at least ensure they are the same. $config = ConfigFactory::getDefaultInstance()->makeConfig('CirrusSearch'); } parent::__construct($conn, $user, $config->get('CirrusSearchSlowSearch')); $this->config = $config; $this->offset = min($offset, self::MAX_OFFSET); $this->limit = $limit; $this->namespaces = $namespaces; $this->indexBaseName = $index ?: $config->getWikiId(); $this->language = $config->get('ContLang'); $this->escaper = new Escaper($config->get('LanguageCode'), $config->get('CirrusSearchAllowLeadingWildcard')); $this->searchContext = new SearchContext($this->config); }
/** * This is really private. */ public function sendDocuments(Type $type, $messagePrefix, $documents) { try { $type->addDocuments($documents); } catch (ExceptionInterface $e) { $errorType = get_class($e); $message = ElasticsearchIntermediary::extractMessage($e); $this->outputIndented($messagePrefix . "Error adding documents in bulk. Retrying as singles. Error type is '{$errorType}' and message is: {$message}"); foreach ($documents as $document) { // Continue using the bulk api because we're used to it. $type->addDocuments(array($document)); } } }
protected function __construct(Connection $conn) { parent::__construct($conn, null, null); $this->linkCountMultiSearch = new \Elastica\Multi\Search($conn->getClient()); $this->linkCountClosures = array(); }
/** * These values end up serialized into Avro which has strict typing * requirements. float !== int !== string. * * @param float $took Number of milliseconds the request took * @return array */ private function buildLogContext($took) { $client = $this->connection->getClient(); $query = $client->getLastRequest(); $result = $client->getLastResponse(); $params = $this->logContext; $this->logContext = array(); $params += array('tookMs' => intval($took), 'source' => self::getExecutionContext(), 'executor' => self::getExecutionId(), 'identity' => self::generateIdentToken()); if ($result) { $queryData = $query->getData(); $resultData = $result->getData(); $index = explode('/', $query->getPath()); $params['index'] = $index[0]; if (isset($resultData['took'])) { $elasticTook = $resultData['took']; $params['elasticTookMs'] = intval($elasticTook); } if (isset($resultData['hits']['total'])) { $params['hitsTotal'] = intval($resultData['hits']['total']); } if (isset($resultData['hits']['hits'])) { $num = count($resultData['hits']['hits']); $offset = isset($queryData['from']) ? $queryData['from'] : 0; $params['hitsReturned'] = $num; $params['hitsOffset'] = intval($offset); } if ($this->_isset($queryData, array('query', 'filtered', 'filter', 'terms', 'namespace'))) { $namespaces = $queryData['query']['filtered']['filter']['terms']['namespace']; $params['namespaces'] = array_map('intval', $namespaces); } if (isset($resultData['suggest']['suggest'][0]['options'][0]['text'])) { $params['suggestion'] = $resultData['suggest']['suggest'][0]['options'][0]['text']; } } if (self::$logContexts === null) { DeferredUpdates::addCallableUpdate(function () { ElasticsearchIntermediary::reportLogContexts(); }); self::$logContexts = array(); } self::$logContexts[] = $params; return $params; }
public function execute() { global $wgPoolCounterConf, $wgLanguageCode, $wgCirrusSearchPhraseSuggestUseText, $wgCirrusSearchPrefixSearchStartsWithAnyWord, $wgCirrusSearchBannedPlugins, $wgCirrusSearchOptimizeIndexForExperimentalHighlighter, $wgCirrusSearchMaxShardsPerNode, $wgCirrusSearchRefreshInterval; // Make sure we don't flood the pool counter unset($wgPoolCounterConf['CirrusSearch-Search']); // Set the timeout for maintenance actions $this->setConnectionTimeout(); $utils = new ConfigUtils($this->getConnection()->getClient(), $this); $this->indexType = $this->getOption('indexType'); $this->startOver = $this->getOption('startOver', false); $this->indexBaseName = $this->getOption('baseName', wfWikiId()); $this->reindexAndRemoveOk = $this->getOption('reindexAndRemoveOk', false); $this->reindexProcesses = $this->getOption('reindexProcesses', wfIsWindows() ? 1 : 5); $this->reindexAcceptableCountDeviation = Util::parsePotentialPercent($this->getOption('reindexAcceptableCountDeviation', '5%')); $this->reindexChunkSize = $this->getOption('reindexChunkSize', 100); $this->reindexRetryAttempts = $this->getOption('reindexRetryAttempts', 5); $this->printDebugCheckConfig = $this->getOption('debugCheckConfig', false); $this->langCode = $wgLanguageCode; $this->prefixSearchStartsWithAny = $wgCirrusSearchPrefixSearchStartsWithAnyWord; $this->phraseSuggestUseText = $wgCirrusSearchPhraseSuggestUseText; $this->bannedPlugins = $wgCirrusSearchBannedPlugins; $this->optimizeIndexForExperimentalHighlighter = $wgCirrusSearchOptimizeIndexForExperimentalHighlighter; $this->maxShardsPerNode = isset($wgCirrusSearchMaxShardsPerNode[$this->indexType]) ? $wgCirrusSearchMaxShardsPerNode[$this->indexType] : 'unlimited'; $this->refreshInterval = $wgCirrusSearchRefreshInterval; try { $indexTypes = $this->getConnection()->getAllIndexTypes(); if (!in_array($this->indexType, $indexTypes)) { $this->error('indexType option must be one of ' . implode(', ', $indexTypes), 1); } $utils->checkElasticsearchVersion(); $this->availablePlugins = $utils->scanAvailablePlugins($this->bannedPlugins); if ($this->getOption('justCacheWarmers', false)) { $this->validateCacheWarmers(); return; } if ($this->getOption('justAllocation', false)) { $this->validateShardAllocation(); return; } $this->indexIdentifier = $utils->pickIndexIdentifierFromOption($this->getOption('indexIdentifier', 'current'), $this->getIndexTypeName()); $this->analysisConfigBuilder = $this->pickAnalyzer($this->langCode, $this->availablePlugins); $this->validateIndex(); $this->validateAnalyzers(); $this->validateMapping(); $this->validateCacheWarmers(); $this->validateAlias(); $this->updateVersions(); $this->indexNamespaces(); } catch (\Elastica\Exception\Connection\HttpException $e) { $message = $e->getMessage(); $this->output("\nUnexpected Elasticsearch failure.\n"); $this->error("Http error communicating with Elasticsearch: {$message}.\n", 1); } catch (\Elastica\Exception\ExceptionInterface $e) { $type = get_class($e); $message = ElasticsearchIntermediary::extractMessage($e); $trace = $e->getTraceAsString(); $this->output("\nUnexpected Elasticsearch failure.\n"); $this->error("Elasticsearch failed in an unexpected way. This is always a bug in CirrusSearch.\n" . "Error type: {$type}\n" . "Message: {$message}\n" . "Trace:\n" . $trace, 1); } }
/** * Constructor * @param int $limit Limit the results to this many * @param SearchConfig Configuration settings * @param int[]|null $namespaces Array of namespace numbers to search or null to search all namespaces. * @param User|null $user user for which this search is being performed. Attached to slow request logs. * @param string|boolean $index Base name for index to search from, defaults to wfWikiId() */ public function __construct(Connection $conn, $limit, SearchConfig $config = null, array $namespaces = null, User $user = null, $index = false) { if (is_null($config)) { // @todo connection has an embeded config ... reuse that? somehow should // at least ensure they are the same. $config = ConfigFactory::getDefaultInstance()->makeConfig('CirrusSearch'); } parent::__construct($conn, $user, $config->get('CirrusSearchSlowSearch')); $this->config = $config; $this->limit = $limit; $this->indexBaseName = $index ?: $config->getWikiId(); $this->searchContext = new SearchContext($this->config, $namespaces); }
/** * @return Status */ public function validate() { $this->outputIndented("Validating mappings..."); if ($this->optimizeIndexForExperimentalHighlighter && !in_array('experimental highlighter', $this->availablePlugins)) { $this->output("impossible!\n"); return Status::newFatal(new RawMessage("wgCirrusSearchOptimizeIndexForExperimentalHighlighter is set to true but the " . "'experimental highlighter' plugin is not installed on all hosts.")); } $requiredMappings = $this->mappingConfig; if (!$this->checkMapping($requiredMappings)) { /** @var Mapping[] $actions */ $actions = array(); // TODO Conflict resolution here might leave old portions of mappings foreach ($this->types as $typeName => $type) { $action = new Mapping($type); foreach ($requiredMappings[$typeName] as $key => $value) { $action->setParam($key, $value); } $actions[] = $action; } try { // @todo Use $action->send(array('master_timeout' => ...)) // after updating to version of Elastica library that supports it. // See https://github.com/ruflin/Elastica/pull/1004 foreach ($actions as $action) { $action->getType()->request('_mapping', Request::PUT, $action->toArray(), array('master_timeout' => $this->masterTimeout)); } $this->output("corrected\n"); } catch (ExceptionInterface $e) { $this->output("failed!\n"); $message = ElasticsearchIntermediary::extractMessage($e); return Status::newFatal(new RawMessage("Couldn't update mappings. Here is elasticsearch's error message: {$message}\n")); } } return Status::newGood(); }
public function execute() { global $wgLanguageCode, $wgCirrusSearchBannedPlugins, $wgCirrusSearchRefreshInterval, $wgPoolCounterConf, $wgCirrusSearchMasterTimeout; $this->masterTimeout = $this->getOption('masterTimeout', $wgCirrusSearchMasterTimeout); $this->indexTypeName = Connection::TITLE_SUGGEST_TYPE; // Check that all shards and replicas settings are set try { $this->getShardCount(); $this->getReplicaCount(); } catch (\Exception $e) { $this->error("Failed to get shard count and replica count information: {$e->getMessage()}", 1); } // Make sure we don't flood the pool counter unset($wgPoolCounterConf['CirrusSearch-Search']); // Set the timeout for maintenance actions $this->setConnectionTimeout(); $this->indexBaseName = $this->getOption('baseName', wfWikiId()); $this->indexChunkSize = $this->getOption('indexChunkSize', 100); $this->indexRetryAttempts = $this->getOption('reindexRetryAttempts', 5); $this->optimizeIndex = $this->getOption('optimize', false); $this->withGeo = $this->getOption('with-geo', false); $this->utils = new ConfigUtils($this->getClient(), $this); $this->langCode = $wgLanguageCode; $this->bannedPlugins = $wgCirrusSearchBannedPlugins; $this->utils->checkElasticsearchVersion(); $this->maxShardsPerNode = isset($wgCirrusSearchMaxShardsPerNode[$this->indexTypeName]) ? $wgCirrusSearchMaxShardsPerNode[$this->indexTypeName] : 'unlimited'; $this->refreshInterval = $wgCirrusSearchRefreshInterval; try { if (!$this->canWrite()) { $this->error('Index/Cluster is frozen. Giving up.', 1); } $oldIndexIdentifier = $this->utils->pickIndexIdentifierFromOption('current', $this->getIndexTypeName()); $this->oldIndex = $this->getConnection()->getIndex($this->indexBaseName, $this->indexTypeName, $oldIndexIdentifier); $this->indexIdentifier = $this->utils->pickIndexIdentifierFromOption('now', $this->getIndexTypeName()); $this->availablePlugins = $this->utils->scanAvailablePlugins($this->bannedPlugins); $this->analysisConfigBuilder = $this->pickAnalyzer($this->langCode, $this->availablePlugins); $this->createIndex(); $this->indexData(); if ($this->optimizeIndex) { $this->optimize(); } $this->enableReplicas(); $this->validateAlias(); $this->updateVersions(); $this->deleteOldIndex(); $this->output("done.\n"); } catch (\Elastica\Exception\Connection\HttpException $e) { $message = $e->getMessage(); $this->output("\nUnexpected Elasticsearch failure.\n"); $this->error("Http error communicating with Elasticsearch: {$message}.\n", 1); } catch (\Elastica\Exception\ExceptionInterface $e) { $type = get_class($e); $message = ElasticsearchIntermediary::extractMessage($e); $trace = $e->getTraceAsString(); $this->output("\nUnexpected Elasticsearch failure.\n"); $this->error("Elasticsearch failed in an unexpected way. This is always a bug in CirrusSearch.\n" . "Error type: {$type}\n" . "Message: {$message}\n" . "Trace:\n" . $trace, 1); } $this->getIndex()->refresh(); }
public function execute() { // Make sure we don't flood the pool counter global $wgPoolCounterConf; unset($wgPoolCounterConf['Flow-Search']); // Sets all of this class' properties that will be passed to validators $this->setProperties(); // Set the timeout for maintenance actions $this->setConnectionTimeout(); try { $indexTypes = $this->getAllIndices(); if (!in_array($this->indexType, $indexTypes)) { $this->error('indexType option must be one of ' . implode(', ', $indexTypes), 1); } $this->utils->checkElasticsearchVersion(); $validators = $this->getValidators(); foreach ($validators as $validator) { $status = $validator->validate(); if (!$status->isOK()) { $this->error($status->getMessage()->text(), 1); } } // $this->updateVersions(); // @todo: might need this some day? (see CirrusSearch's UpdateOneSearchIndexConfig::updateVersions) } catch (\Elastica\Exception\Connection\HttpException $e) { $message = $e->getMessage(); $this->output("\nUnexpected Elasticsearch failure.\n"); $this->error("Http error communicating with Elasticsearch: {$message}.\n", 1); } catch (\Elastica\Exception\ExceptionInterface $e) { $type = get_class($e); $message = ElasticsearchIntermediary::extractMessage($e); $trace = $e->getTraceAsString(); $this->output("\nUnexpected Elasticsearch failure.\n"); $this->error("Elasticsearch failed in an unexpected way. This is always a bug in FlowSearch.\n" . "Error type: {$type}\n" . "Message: {$message}\n" . "Trace:\n" . $trace, 1); } }
/** * Constructor */ public function __construct(Connection $conn) { parent::__construct($conn, null, 0); }