Handles reads, deletes and configurations of an index
public function testQuery()
{
$client = $this->_getClient();
$index = new Index($client, 'test');
$index->create(array(), true);
$type = new Type($index, 'constant_score');
$doc = new Document(1, array('id' => 1, 'email' => '*****@*****.**', 'username' => 'hans'));
$type->addDocument($doc);
$doc = new Document(2, array('id' => 2, 'email' => '*****@*****.**', 'username' => 'emil'));
$type->addDocument($doc);
$doc = new Document(3, array('id' => 3, 'email' => '*****@*****.**', 'username' => 'ruth'));
$type->addDocument($doc);
// Refresh index
$index->refresh();
$boost = 1.3;
$query_match = new MatchAll();
$query = new ConstantScore();
$query->setQuery($query_match);
$query->setBoost($boost);
$expectedArray = array('constant_score' => array('query' => $query_match->toArray(), 'boost' => $boost));
$this->assertEquals($expectedArray, $query->toArray());
$resultSet = $type->search($query);
$results = $resultSet->getResults();
$this->assertEquals($resultSet->count(), 3);
$this->assertEquals($results[1]->getScore(), 1);
}
/**
* @param array $dataSet
*
* @return bool
*/
public function write(array $dataSet)
{
$type = $this->index->getType($this->type);
$type->updateDocuments($this->createDocuments($dataSet));
$response = $type->getIndex()->refresh();
return $response->isOk();
}
protected function tearDown()
{
parent::tearDown();
if ($this->_index instanceof Index) {
$this->_index->delete();
}
}
public function testGetDocument()
{
$type = self::$index->getType('message');
$document = $type->getDocument(1);
$message = $this->marshaler->unmarshal($document);
$this->assertTrue($this->message->equals($message));
}
/**
* @group functional
*/
public function testSearch()
{
$client = $this->_getClient();
$index = new Index($client, 'test');
$index->create(array(), true);
$type = new Type($index, 'helloworld');
$doc = new Document(1, array('id' => 1, 'email' => '*****@*****.**', 'username' => 'hans', 'test' => array('2', '3', '5')));
$type->addDocument($doc);
$doc = new Document(2, array('id' => 2, 'email' => '*****@*****.**', 'username' => 'emil', 'test' => array('1', '3', '6')));
$type->addDocument($doc);
$doc = new Document(3, array('id' => 3, 'email' => '*****@*****.**', 'username' => 'ruth', 'test' => array('2', '3', '7')));
$type->addDocument($doc);
// Refresh index
$index->refresh();
$boolQuery = new BoolQuery();
$termQuery1 = new Term(array('test' => '2'));
$boolQuery->addMust($termQuery1);
$resultSet = $type->search($boolQuery);
$this->assertEquals(2, $resultSet->count());
$termQuery2 = new Term(array('test' => '5'));
$boolQuery->addMust($termQuery2);
$resultSet = $type->search($boolQuery);
$this->assertEquals(1, $resultSet->count());
$termQuery3 = new Term(array('username' => 'hans'));
$boolQuery->addMust($termQuery3);
$resultSet = $type->search($boolQuery);
$this->assertEquals(1, $resultSet->count());
$termQuery4 = new Term(array('username' => 'emil'));
$boolQuery->addMust($termQuery4);
$resultSet = $type->search($boolQuery);
$this->assertEquals(0, $resultSet->count());
}
/**
* @group functional
*/
public function testSearch()
{
$client = $this->_getClient();
$index = new Index($client, 'test');
$index->create(array(), true);
$index->getSettings()->setNumberOfReplicas(0);
//$index->getSettings()->setNumberOfShards(1);
$type = new Type($index, 'helloworldmlt');
$mapping = new Mapping($type, array('email' => array('store' => 'yes', 'type' => 'string', 'index' => 'analyzed'), 'content' => array('store' => 'yes', 'type' => 'string', 'index' => 'analyzed')));
$mapping->setSource(array('enabled' => false));
$type->setMapping($mapping);
$doc = new Document(1000, array('email' => '*****@*****.**', 'content' => 'This is a sample post. Hello World Fuzzy Like This!'));
$type->addDocument($doc);
$doc = new Document(1001, array('email' => '*****@*****.**', 'content' => 'This is a fake nospam email address for gmail'));
$type->addDocument($doc);
// Refresh index
$index->refresh();
$mltQuery = new MoreLikeThis();
$mltQuery->setLike('fake gmail sample');
$mltQuery->setFields(array('email', 'content'));
$mltQuery->setMaxQueryTerms(3);
$mltQuery->setMinDocFrequency(1);
$mltQuery->setMinTermFrequency(1);
$query = new Query();
$query->setQuery($mltQuery);
$resultSet = $type->search($query);
$resultSet->getResponse()->getData();
$this->assertEquals(2, $resultSet->count());
}
/**
* Match a document to percolator queries
*
* @param \Elastica\Document $doc
* @param string|\Elastica\Query|\Elastica\Query\AbstractQuery $query Not implemented yet
* @return \Elastica\Response
*/
public function matchDoc(Document $doc, $query = null)
{
$path = $this->_index->getName() . '/type/_percolate';
$data = array('doc' => $doc->getData());
$response = $this->getIndex()->getClient()->request($path, Request::GET, $data);
$data = $response->getData();
return $data['matches'];
}
public function createMapping()
{
// Define mapping
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($this->trackIndex->getType('track'));
// Set mapping
$mapping->setProperties(['id' => ['type' => 'integer', 'include_in_all' => false], 'album' => ['type' => 'object', 'properties' => ['id' => ['type' => 'integer', 'include_in_all' => true], 'title' => ['type' => 'string', 'include_in_all' => true]]], 'name' => ['type' => 'string', 'include_in_all' => true], 'name_not_analyzed' => ['type' => 'string', "index" => "not_analyzed"], 'playList' => ['type' => 'nested', 'properties' => ['id' => ['type' => 'integer', 'include_in_all' => true], 'name' => ['type' => 'string', 'include_in_all' => true]]], 'genre' => ['type' => 'object', 'properties' => ['id' => ['type' => 'integer', 'include_in_all' => true], 'name' => ['type' => 'string', 'include_in_all' => true]]], 'mediaType' => ['type' => 'object', 'properties' => ['id' => ['type' => 'integer', 'include_in_all' => true], 'name' => ['type' => 'string', 'include_in_all' => true]]], 'composer' => ['type' => 'string', 'include_in_all' => true]]);
// Send mapping to type
$mapping->send();
}
/**
* @return Index
*/
private function getIndex()
{
if (null === $this->index) {
$this->index = $this->client->getIndex($this->indexName);
if (!$this->index->exists()) {
$this->index->create();
}
}
return $this->index;
}
public function testQuery()
{
$query = new SimpleQueryString("gibson +sg +-faded", array("make", "model"));
$results = $this->_index->search($query);
$this->assertEquals(2, $results->getTotalHits());
$query->setFields(array("model"));
$results = $this->_index->search($query);
// We should not get any hits, since the "make" field was not included in the query.
$this->assertEquals(0, $results->getTotalHits());
}
public function testSuggestNoResults()
{
$termSuggest = new Term('suggest1', '_all');
$termSuggest->setText('Foobar')->setSize(4);
$result = $this->_index->search($termSuggest);
$this->assertEquals(1, $result->countSuggests());
// Assert that no suggestions were returned
$suggests = $result->getSuggests();
$this->assertEquals(0, sizeof($suggests['suggest1'][0]['options']));
}
/**
* @param \Elastica\Index $index
* @param string $mappingName
* @param array $mappingData
*
* @return void
*/
protected function sendMapping(Index $index, $mappingName, array $mappingData)
{
$type = $index->getType($mappingName);
$this->messenger->info(sprintf('Send mapping type "%s" (index: "%s")', $mappingName, $index->getName()));
$mapping = new Mapping($type);
foreach ($mappingData as $key => $value) {
$mapping->setParam($key, $value);
}
$mapping->send();
}
/**
* @expectedException \Elastica\Exception\ResponseException
*/
public function testInvalidElasticRequest()
{
$connection = new Connection();
$connection->setHost('localhost');
$connection->setPort(9500);
$connection->setTransport('Thrift');
$client = new Client();
$client->addConnection($connection);
$index = new Index($client, 'missing_index');
$index->getStatus();
}
/**
* @group functional
* @expectedException \Elastica\Exception\ResponseException
*/
public function testInvalidElasticRequest()
{
$this->_checkPlugin();
$connection = new Connection();
$connection->setHost($this->_getHost());
$connection->setPort(9500);
$connection->setTransport('Thrift');
$client = $this->_getClient();
$client->addConnection($connection);
$index = new Index($client, 'missing_index');
$index->getStatus();
}
protected function _waitForAllocation(Index $index)
{
do {
$settings = $index->getStatus()->get();
$allocated = true;
foreach ($settings['shards'] as $shard) {
if ($shard[0]['routing']['state'] != 'STARTED') {
$allocated = false;
}
}
} while (!$allocated);
}
public function testQuery()
{
$query = new Query();
$match = new Match();
$match->setField('make', 'ford');
$query->setQuery($match);
$filter = new Term();
$filter->setTerm('color', 'green');
$query->setPostFilter($filter);
$results = $this->_index->search($query);
$this->assertEquals(1, $results->getTotalHits());
}
/**
* @return Status
*/
public function validate()
{
$this->outputIndented("Validating analyzers...");
$settings = $this->index->getSettings()->get();
$requiredAnalyzers = $this->analysisConfigBuilder->buildConfig();
if ($this->checkConfig($settings['analysis'], $requiredAnalyzers)) {
$this->output("ok\n");
} else {
$this->output("cannot correct\n");
return Status::newFatal(new RawMessage("This script encountered an index difference that requires that the index be\n" . "copied, indexed to, and then the old index removed. Re-run this script with the\n" . "--reindexAndRemoveOk --indexIdentifier=now parameters to do this."));
}
return Status::newGood();
}
/**
* @group unit
*/
public function testSetType()
{
$document = new Document();
$document->setType('type');
$this->assertEquals('type', $document->getType());
$index = new Index($this->_getClient(), 'index');
$type = $index->getType('type');
$document->setIndex('index2');
$this->assertEquals('index2', $document->getIndex());
$document->setType($type);
$this->assertEquals('index', $document->getIndex());
$this->assertEquals('type', $document->getType());
}
/**
* Match a document to percolator queries
*
* @param \Elastica\Document $doc
* @param string|\Elastica\Query|\Elastica\Query\AbstractQuery $query Query to filter the data
* @return \Elastica\Response
*/
public function matchDoc(Document $doc, $query = null)
{
$path = $this->_index->getName() . '/type/_percolate';
$data = array('doc' => $doc->getData());
// Add query to filter results after percolation
if ($query) {
$query = Query::create($query);
$data['query'] = $query->getQuery();
}
$response = $this->getIndex()->getClient()->request($path, Request::GET, $data);
$data = $response->getData();
return $data['matches'];
}
/**
* @return Status
*/
public function validate()
{
$this->outputIndented("\tValidating number of shards...");
$settings = $this->index->getSettings()->get();
$actualShardCount = $settings['number_of_shards'];
if ($actualShardCount == $this->shardCount) {
$this->output("ok\n");
} else {
$this->output("is {$actualShardCount} but should be " . $this->shardCount . "...cannot correct!\n");
return Status::newFatal(new RawMessage("Number of shards is incorrect and cannot be changed without a rebuild. You can solve this\n" . "problem by running this program again with either --startOver or --reindexAndRemoveOk. Make\n" . "sure you understand the consequences of either choice.. This script will now continue to\n" . "validate everything else."));
}
return Status::newGood();
}
/**
* Match a document to percolator queries
*
* @param \Elastica\Document $doc
* @param string|\Elastica\Query|\Elastica\Query\AbstractQuery $query Query to filter the percolator queries which
* are executed.
* @param string $type
* @return array With matching registered queries.
*/
public function matchDoc(Document $doc, $query = null, $type = 'type')
{
$path = $this->_index->getName() . '/' . $type . '/_percolate';
$data = array('doc' => $doc->getData());
// Add query to filter the percolator queries which are executed.
if ($query) {
$query = Query::create($query);
$data['query'] = $query->getQuery();
}
$response = $this->getIndex()->getClient()->request($path, Request::GET, $data);
$data = $response->getData();
return $data['matches'];
}
/**
* @return Status
*/
public function validate()
{
$this->outputIndented("\tValidating replica range...");
$settings = $this->index->getSettings()->get();
$actualReplicaCount = isset($settings['auto_expand_replicas']) ? $settings['auto_expand_replicas'] : 'false';
if ($actualReplicaCount == $this->replicaCount) {
$this->output("ok\n");
} else {
$this->output("is {$actualReplicaCount} but should be " . $this->replicaCount . '...');
$this->index->getSettings()->set(array('auto_expand_replicas' => $this->replicaCount));
$this->output("corrected\n");
}
return Status::newGood();
}
/**
* @group unit
*/
public function testToArrayFromReference()
{
$client = $this->_getClient();
$index = new Index($client, 'test');
$type = new Type($index, 'helloworld');
$field = 'image';
$query = new Image();
$query->setFieldFeature($field, 'CEDD');
$query->setFieldHash($field, 'BIT_SAMPLING');
$query->setFieldBoost($field, 100);
$query->setImageByReference($field, $index->getName(), $type->getName(), 10);
$jsonString = '{"image":{"image":{"feature":"CEDD","hash":"BIT_SAMPLING","boost":100,"index":"test","type":"helloworld","id":10,"path":"image"}}}';
$this->assertEquals($jsonString, json_encode($query->toArray()));
}
public function testPhraseSuggest()
{
$suggest = new Suggest();
$phraseSuggest = new Phrase('suggest1', 'text');
$phraseSuggest->setText("elasticsearch is bansai coor");
$phraseSuggest->setAnalyzer("simple")->setHighlight("<suggest>", "</suggest>")->setStupidBackoffSmoothing(0.4);
$phraseSuggest->addCandidateGenerator(new DirectGenerator("text"));
$suggest->addSuggestion($phraseSuggest);
$result = $this->_index->search($suggest);
$suggests = $result->getSuggests();
// 3 suggestions should be returned: One in which both misspellings are corrected, and two in which only one misspelling is corrected.
$this->assertEquals(3, sizeof($suggests['suggest1'][0]['options']));
$this->assertEquals("elasticsearch is <suggest>bonsai cool</suggest>", $suggests['suggest1'][0]['options'][0]['highlighted']);
$this->assertEquals("elasticsearch is bonsai cool", $suggests['suggest1'][0]['options'][0]['text']);
}
public function getType($type)
{
if (isset($this->typeCache[$type])) {
return $this->typeCache[$type];
}
return $this->typeCache[$type] = parent::getType($type);
}
/**
* @actionInfo(name='Бэкап данных')
*/
public function backupAction()
{
$this->folderName = __DIR__ . '/backup/';
if (!$this->elasticaIndex->exists()) {
$this->log->error('Индекс для бэкапа отсутствует: {indexName}', ['indexName' => $this->indexName]);
return;
}
$this->checkFileName();
$this->checkBackupFolder();
$this->log->info('Всё ок, бекапим {indexName} в {fileName}', ['indexName' => $this->indexName, 'fileName' => $this->fileName]);
$this->log->info('Параметры бэкапа: sizePerShard={sizePerShard}', ['sizePerShard' => $this->sizePerShard]);
$scanAndScroll = $this->getScanAndScroll();
foreach ($scanAndScroll as $resultSet) {
$buffer = [];
/* @var \Elastica\ResultSet $resultSet */
$results = $resultSet->getResults();
foreach ($results as $result) {
$item = [];
$item['_id'] = $result->getId();
$item['_source'] = $result->getSource();
$buffer[] = json_encode($item, JSON_UNESCAPED_UNICODE);
}
$fileBody = implode(PHP_EOL, $buffer);
if (file_put_contents($this->fileName, $fileBody, FILE_APPEND)) {
$countDocuments = count($results);
$this->log->info('Сохранили {countDocuments} записей', ['countDocuments' => $countDocuments]);
} else {
$this->log->error('Ошибка записи данных');
die;
}
}
}
public function testSearch()
{
$client = $this->_getClient();
$index = new Index($client, 'test');
$index->create(array(), true);
$index->getSettings()->setNumberOfReplicas(0);
//$index->getSettings()->setNumberOfShards(1);
$type = new Type($index, 'helloworld');
$doc = new Document(1, array('email' => '*****@*****.**', 'username' => 'hanswurst', 'test' => array('2', '3', '5')));
$type->addDocument($doc);
// Refresh index
$index->refresh();
$queryString = new QueryString('test*');
$resultSet = $type->search($queryString);
$this->assertEquals(1, $resultSet->count());
}
/**
* @param array $dataSet
*
* @throws \Spryker\Zed\Collector\Business\Exporter\Exception\InvalidDataSetException
*
* @return bool
*/
public function delete(array $dataSet)
{
if ($this->hasIntegerKeys($dataSet)) {
throw new InvalidDataSetException();
}
try {
$documents = [];
foreach ($dataSet as $key => $value) {
$documents[] = $this->index->getType($this->type)->getDocument($key);
}
$response = $this->index->deleteDocuments($documents);
$this->index->flush(true);
return $response->isOk();
} catch (\Exception $exception) {
return true;
}
}
/**
* Percolating an existing document.
*
* @param string $id
* @param string $type
* @param string|\Elastica\Query|\Elastica\Query\AbstractQuery $query Query to filter the percolator queries which
* are executed.
* @param array $params Supports setting additional request body options to the percolate request.
* [ Percolator::EXTRA_FILTER,
* Percolator::EXTRA_QUERY,
* Percolator::EXTRA_SIZE,
* Percolator::EXTRA_TRACK_SCORES,
* Percolator::EXTRA_SORT,
* Percolator::EXTRA_FACETS,
* Percolator::EXTRA_AGGS,
* Percolator::EXTRA_HIGHLIGHT ]
*
* @return array With matching registered queries.
*/
public function matchExistingDoc($id, $type, $query = null, $params = array())
{
$id = urlencode($id);
$path = $this->_index->getName() . '/' . $type . '/' . $id . '/_percolate';
$data = array();
$this->_applyAdditionalRequestBodyOptions($params, $data);
return $this->_percolate($path, $query, $data, $params);
}
protected function setUp()
{
/** @var ElasticaService elasticaService */
$elasticaService = $this->getContainer()->get("revinate_search.elasticsearch_service");
$this->elasticaClient = $elasticaService->getInstance();
$this->index = new \Elastica\Index($this->elasticaClient, View::INDEX_NAME);
if (!$this->index->exists()) {
$this->index->create(array("index.number_of_replicas" => "0", "index.number_of_shards" => "1"));
$this->type = new \Elastica\Type($this->index, View::INDEX_TYPE);
$mappingJson = json_decode(file_get_contents(__DIR__ . "/../data/es/mapping.json"), true);
$mapping = new \Elastica\Type\Mapping($this->type, $mappingJson['properties']);
$this->type->setMapping($mapping);
} else {
$this->type = new \Elastica\Type($this->index, View::INDEX_TYPE);
}
$this->timeSeriesIndex = new \Elastica\Index($this->elasticaClient, StatusLog::INDEX_NAME . self::$timeSeriesTestDateSuffix);
if (!$this->timeSeriesIndex->exists()) {
$this->timeSeriesIndex->create(array("index.number_of_replicas" => "0", "index.number_of_shards" => "1"));
$this->timeSeriesType = new \Elastica\Type($this->timeSeriesIndex, StatusLog::INDEX_TYPE);
$mappingJson = json_decode(file_get_contents(__DIR__ . "/../data/es/statusLogMapping.json"), true);
$mapping = new \Elastica\Type\Mapping($this->timeSeriesType, $mappingJson['properties']);
$this->timeSeriesType->setMapping($mapping);
} else {
$this->timeSeriesType = new \Elastica\Type($this->timeSeriesIndex, StatusLog::INDEX_TYPE);
}
}
