public function __construct(Connection $conn, array $flags = array())
{
parent::__construct($conn, null, null);
if (in_array('same-cluster', $flags)) {
$this->writeToClusterName = $this->connection->getClusterName();
}
}
/**
* @return Status
*/
public function validate()
{
$this->outputIndented("Validating mappings...");
if ($this->optimizeIndexForExperimentalHighlighter && !in_array('experimental highlighter', $this->availablePlugins)) {
$this->output("impossible!\n");
return Status::newFatal(new RawMessage("wgCirrusSearchOptimizeIndexForExperimentalHighlighter is set to true but the " . "'experimental highlighter' plugin is not installed on all hosts."));
}
$requiredMappings = $this->mappingConfig;
if (!$this->checkMapping($requiredMappings)) {
/** @var Mapping[] $actions */
$actions = array();
// TODO Conflict resolution here might leave old portions of mappings
foreach ($this->types as $typeName => $type) {
$action = new Mapping($type);
foreach ($requiredMappings[$typeName] as $key => $value) {
$action->setParam($key, $value);
}
$actions[] = $action;
}
try {
foreach ($actions as $action) {
$action->send();
}
$this->output("corrected\n");
} catch (ExceptionInterface $e) {
$this->output("failed!\n");
$message = ElasticsearchIntermediary::extractMessage($e);
return Status::newFatal(new RawMessage("Couldn't update mappings. Here is elasticsearch's error message: {$message}\n"));
}
}
return Status::newGood();
}
/**
* @var Connection
*/
public function __construct(Connection $conn)
{
parent::__construct($conn, null, null);
$this->log = LoggerFactory::getInstance('CirrusSearch');
$this->failedLog = LoggerFactory::getInstance('CirrusSearchChangeFailed');
}
/**
* Constructor
* @param int $offset Offset the results by this much
* @param int $limit Limit the results to this many
* @param SearchConfig Configuration settings
* @param int[]|null $namespaces Array of namespace numbers to search or null to search all namespaces.
* @param User|null $user user for which this search is being performed. Attached to slow request logs.
* @param string|boolean $index Base name for index to search from, defaults to wfWikiId()
*/
public function __construct(Connection $conn, $offset, $limit, SearchConfig $config = null, array $namespaces = null, User $user = null, $index = false)
{
if (is_null($config)) {
// @todo connection has an embeded config ... reuse that? somehow should
// at least ensure they are the same.
$config = ConfigFactory::getDefaultInstance()->makeConfig('CirrusSearch');
}
parent::__construct($conn, $user, $config->get('CirrusSearchSlowSearch'));
$this->config = $config;
$this->offset = min($offset, self::MAX_OFFSET);
$this->limit = $limit;
$this->namespaces = $namespaces;
$this->indexBaseName = $index ?: $config->getWikiId();
$this->language = $config->get('ContLang');
$this->escaper = new Escaper($config->get('LanguageCode'), $config->get('CirrusSearchAllowLeadingWildcard'));
$this->searchContext = new SearchContext($this->config);
}
/**
* This is really private.
*/
public function sendDocuments(Type $type, $messagePrefix, $documents)
{
try {
$type->addDocuments($documents);
} catch (ExceptionInterface $e) {
$errorType = get_class($e);
$message = ElasticsearchIntermediary::extractMessage($e);
$this->outputIndented($messagePrefix . "Error adding documents in bulk. Retrying as singles. Error type is '{$errorType}' and message is: {$message}");
foreach ($documents as $document) {
// Continue using the bulk api because we're used to it.
$type->addDocuments(array($document));
}
}
}
protected function __construct(Connection $conn)
{
parent::__construct($conn, null, null);
$this->linkCountMultiSearch = new \Elastica\Multi\Search($conn->getClient());
$this->linkCountClosures = array();
}
/**
* These values end up serialized into Avro which has strict typing
* requirements. float !== int !== string.
*
* @param float $took Number of milliseconds the request took
* @return array
*/
private function buildLogContext($took)
{
$client = $this->connection->getClient();
$query = $client->getLastRequest();
$result = $client->getLastResponse();
$params = $this->logContext;
$this->logContext = array();
$params += array('tookMs' => intval($took), 'source' => self::getExecutionContext(), 'executor' => self::getExecutionId(), 'identity' => self::generateIdentToken());
if ($result) {
$queryData = $query->getData();
$resultData = $result->getData();
$index = explode('/', $query->getPath());
$params['index'] = $index[0];
if (isset($resultData['took'])) {
$elasticTook = $resultData['took'];
$params['elasticTookMs'] = intval($elasticTook);
}
if (isset($resultData['hits']['total'])) {
$params['hitsTotal'] = intval($resultData['hits']['total']);
}
if (isset($resultData['hits']['hits'])) {
$num = count($resultData['hits']['hits']);
$offset = isset($queryData['from']) ? $queryData['from'] : 0;
$params['hitsReturned'] = $num;
$params['hitsOffset'] = intval($offset);
}
if ($this->_isset($queryData, array('query', 'filtered', 'filter', 'terms', 'namespace'))) {
$namespaces = $queryData['query']['filtered']['filter']['terms']['namespace'];
$params['namespaces'] = array_map('intval', $namespaces);
}
if (isset($resultData['suggest']['suggest'][0]['options'][0]['text'])) {
$params['suggestion'] = $resultData['suggest']['suggest'][0]['options'][0]['text'];
}
}
if (self::$logContexts === null) {
DeferredUpdates::addCallableUpdate(function () {
ElasticsearchIntermediary::reportLogContexts();
});
self::$logContexts = array();
}
self::$logContexts[] = $params;
return $params;
}
public function execute()
{
global $wgPoolCounterConf, $wgLanguageCode, $wgCirrusSearchPhraseSuggestUseText, $wgCirrusSearchPrefixSearchStartsWithAnyWord, $wgCirrusSearchBannedPlugins, $wgCirrusSearchOptimizeIndexForExperimentalHighlighter, $wgCirrusSearchMaxShardsPerNode, $wgCirrusSearchRefreshInterval;
// Make sure we don't flood the pool counter
unset($wgPoolCounterConf['CirrusSearch-Search']);
// Set the timeout for maintenance actions
$this->setConnectionTimeout();
$utils = new ConfigUtils($this->getConnection()->getClient(), $this);
$this->indexType = $this->getOption('indexType');
$this->startOver = $this->getOption('startOver', false);
$this->indexBaseName = $this->getOption('baseName', wfWikiId());
$this->reindexAndRemoveOk = $this->getOption('reindexAndRemoveOk', false);
$this->reindexProcesses = $this->getOption('reindexProcesses', wfIsWindows() ? 1 : 5);
$this->reindexAcceptableCountDeviation = Util::parsePotentialPercent($this->getOption('reindexAcceptableCountDeviation', '5%'));
$this->reindexChunkSize = $this->getOption('reindexChunkSize', 100);
$this->reindexRetryAttempts = $this->getOption('reindexRetryAttempts', 5);
$this->printDebugCheckConfig = $this->getOption('debugCheckConfig', false);
$this->langCode = $wgLanguageCode;
$this->prefixSearchStartsWithAny = $wgCirrusSearchPrefixSearchStartsWithAnyWord;
$this->phraseSuggestUseText = $wgCirrusSearchPhraseSuggestUseText;
$this->bannedPlugins = $wgCirrusSearchBannedPlugins;
$this->optimizeIndexForExperimentalHighlighter = $wgCirrusSearchOptimizeIndexForExperimentalHighlighter;
$this->maxShardsPerNode = isset($wgCirrusSearchMaxShardsPerNode[$this->indexType]) ? $wgCirrusSearchMaxShardsPerNode[$this->indexType] : 'unlimited';
$this->refreshInterval = $wgCirrusSearchRefreshInterval;
try {
$indexTypes = $this->getConnection()->getAllIndexTypes();
if (!in_array($this->indexType, $indexTypes)) {
$this->error('indexType option must be one of ' . implode(', ', $indexTypes), 1);
}
$utils->checkElasticsearchVersion();
$this->availablePlugins = $utils->scanAvailablePlugins($this->bannedPlugins);
if ($this->getOption('justCacheWarmers', false)) {
$this->validateCacheWarmers();
return;
}
if ($this->getOption('justAllocation', false)) {
$this->validateShardAllocation();
return;
}
$this->indexIdentifier = $utils->pickIndexIdentifierFromOption($this->getOption('indexIdentifier', 'current'), $this->getIndexTypeName());
$this->analysisConfigBuilder = $this->pickAnalyzer($this->langCode, $this->availablePlugins);
$this->validateIndex();
$this->validateAnalyzers();
$this->validateMapping();
$this->validateCacheWarmers();
$this->validateAlias();
$this->updateVersions();
$this->indexNamespaces();
} catch (\Elastica\Exception\Connection\HttpException $e) {
$message = $e->getMessage();
$this->output("\nUnexpected Elasticsearch failure.\n");
$this->error("Http error communicating with Elasticsearch: {$message}.\n", 1);
} catch (\Elastica\Exception\ExceptionInterface $e) {
$type = get_class($e);
$message = ElasticsearchIntermediary::extractMessage($e);
$trace = $e->getTraceAsString();
$this->output("\nUnexpected Elasticsearch failure.\n");
$this->error("Elasticsearch failed in an unexpected way. This is always a bug in CirrusSearch.\n" . "Error type: {$type}\n" . "Message: {$message}\n" . "Trace:\n" . $trace, 1);
}
}
/**
* Constructor
* @param int $limit Limit the results to this many
* @param SearchConfig Configuration settings
* @param int[]|null $namespaces Array of namespace numbers to search or null to search all namespaces.
* @param User|null $user user for which this search is being performed. Attached to slow request logs.
* @param string|boolean $index Base name for index to search from, defaults to wfWikiId()
*/
public function __construct(Connection $conn, $limit, SearchConfig $config = null, array $namespaces = null, User $user = null, $index = false)
{
if (is_null($config)) {
// @todo connection has an embeded config ... reuse that? somehow should
// at least ensure they are the same.
$config = ConfigFactory::getDefaultInstance()->makeConfig('CirrusSearch');
}
parent::__construct($conn, $user, $config->get('CirrusSearchSlowSearch'));
$this->config = $config;
$this->limit = $limit;
$this->indexBaseName = $index ?: $config->getWikiId();
$this->searchContext = new SearchContext($this->config, $namespaces);
}
/**
* @return Status
*/
public function validate()
{
$this->outputIndented("Validating mappings...");
if ($this->optimizeIndexForExperimentalHighlighter && !in_array('experimental highlighter', $this->availablePlugins)) {
$this->output("impossible!\n");
return Status::newFatal(new RawMessage("wgCirrusSearchOptimizeIndexForExperimentalHighlighter is set to true but the " . "'experimental highlighter' plugin is not installed on all hosts."));
}
$requiredMappings = $this->mappingConfig;
if (!$this->checkMapping($requiredMappings)) {
/** @var Mapping[] $actions */
$actions = array();
// TODO Conflict resolution here might leave old portions of mappings
foreach ($this->types as $typeName => $type) {
$action = new Mapping($type);
foreach ($requiredMappings[$typeName] as $key => $value) {
$action->setParam($key, $value);
}
$actions[] = $action;
}
try {
// @todo Use $action->send(array('master_timeout' => ...))
// after updating to version of Elastica library that supports it.
// See https://github.com/ruflin/Elastica/pull/1004
foreach ($actions as $action) {
$action->getType()->request('_mapping', Request::PUT, $action->toArray(), array('master_timeout' => $this->masterTimeout));
}
$this->output("corrected\n");
} catch (ExceptionInterface $e) {
$this->output("failed!\n");
$message = ElasticsearchIntermediary::extractMessage($e);
return Status::newFatal(new RawMessage("Couldn't update mappings. Here is elasticsearch's error message: {$message}\n"));
}
}
return Status::newGood();
}
public function execute()
{
global $wgLanguageCode, $wgCirrusSearchBannedPlugins, $wgCirrusSearchRefreshInterval, $wgPoolCounterConf, $wgCirrusSearchMasterTimeout;
$this->masterTimeout = $this->getOption('masterTimeout', $wgCirrusSearchMasterTimeout);
$this->indexTypeName = Connection::TITLE_SUGGEST_TYPE;
// Check that all shards and replicas settings are set
try {
$this->getShardCount();
$this->getReplicaCount();
} catch (\Exception $e) {
$this->error("Failed to get shard count and replica count information: {$e->getMessage()}", 1);
}
// Make sure we don't flood the pool counter
unset($wgPoolCounterConf['CirrusSearch-Search']);
// Set the timeout for maintenance actions
$this->setConnectionTimeout();
$this->indexBaseName = $this->getOption('baseName', wfWikiId());
$this->indexChunkSize = $this->getOption('indexChunkSize', 100);
$this->indexRetryAttempts = $this->getOption('reindexRetryAttempts', 5);
$this->optimizeIndex = $this->getOption('optimize', false);
$this->withGeo = $this->getOption('with-geo', false);
$this->utils = new ConfigUtils($this->getClient(), $this);
$this->langCode = $wgLanguageCode;
$this->bannedPlugins = $wgCirrusSearchBannedPlugins;
$this->utils->checkElasticsearchVersion();
$this->maxShardsPerNode = isset($wgCirrusSearchMaxShardsPerNode[$this->indexTypeName]) ? $wgCirrusSearchMaxShardsPerNode[$this->indexTypeName] : 'unlimited';
$this->refreshInterval = $wgCirrusSearchRefreshInterval;
try {
if (!$this->canWrite()) {
$this->error('Index/Cluster is frozen. Giving up.', 1);
}
$oldIndexIdentifier = $this->utils->pickIndexIdentifierFromOption('current', $this->getIndexTypeName());
$this->oldIndex = $this->getConnection()->getIndex($this->indexBaseName, $this->indexTypeName, $oldIndexIdentifier);
$this->indexIdentifier = $this->utils->pickIndexIdentifierFromOption('now', $this->getIndexTypeName());
$this->availablePlugins = $this->utils->scanAvailablePlugins($this->bannedPlugins);
$this->analysisConfigBuilder = $this->pickAnalyzer($this->langCode, $this->availablePlugins);
$this->createIndex();
$this->indexData();
if ($this->optimizeIndex) {
$this->optimize();
}
$this->enableReplicas();
$this->validateAlias();
$this->updateVersions();
$this->deleteOldIndex();
$this->output("done.\n");
} catch (\Elastica\Exception\Connection\HttpException $e) {
$message = $e->getMessage();
$this->output("\nUnexpected Elasticsearch failure.\n");
$this->error("Http error communicating with Elasticsearch: {$message}.\n", 1);
} catch (\Elastica\Exception\ExceptionInterface $e) {
$type = get_class($e);
$message = ElasticsearchIntermediary::extractMessage($e);
$trace = $e->getTraceAsString();
$this->output("\nUnexpected Elasticsearch failure.\n");
$this->error("Elasticsearch failed in an unexpected way. This is always a bug in CirrusSearch.\n" . "Error type: {$type}\n" . "Message: {$message}\n" . "Trace:\n" . $trace, 1);
}
$this->getIndex()->refresh();
}
public function execute()
{
// Make sure we don't flood the pool counter
global $wgPoolCounterConf;
unset($wgPoolCounterConf['Flow-Search']);
// Sets all of this class' properties that will be passed to validators
$this->setProperties();
// Set the timeout for maintenance actions
$this->setConnectionTimeout();
try {
$indexTypes = $this->getAllIndices();
if (!in_array($this->indexType, $indexTypes)) {
$this->error('indexType option must be one of ' . implode(', ', $indexTypes), 1);
}
$this->utils->checkElasticsearchVersion();
$validators = $this->getValidators();
foreach ($validators as $validator) {
$status = $validator->validate();
if (!$status->isOK()) {
$this->error($status->getMessage()->text(), 1);
}
}
// $this->updateVersions(); // @todo: might need this some day? (see CirrusSearch's UpdateOneSearchIndexConfig::updateVersions)
} catch (\Elastica\Exception\Connection\HttpException $e) {
$message = $e->getMessage();
$this->output("\nUnexpected Elasticsearch failure.\n");
$this->error("Http error communicating with Elasticsearch: {$message}.\n", 1);
} catch (\Elastica\Exception\ExceptionInterface $e) {
$type = get_class($e);
$message = ElasticsearchIntermediary::extractMessage($e);
$trace = $e->getTraceAsString();
$this->output("\nUnexpected Elasticsearch failure.\n");
$this->error("Elasticsearch failed in an unexpected way. This is always a bug in FlowSearch.\n" . "Error type: {$type}\n" . "Message: {$message}\n" . "Trace:\n" . $trace, 1);
}
}
/**
* Constructor
*/
public function __construct(Connection $conn)
{
parent::__construct($conn, null, 0);
}
