elasticsearch has for every types as a substructure. This object
represents a type inside a context
The hierarchy is as following: client -> index -> type -> document
/**
* remove the eav deleted products from elasticsearch index
*
* @param array $productIds
*/
public function cleanElasticSearchIndex(array $productIds)
{
foreach ($productIds as $productId) {
try {
$this->_type->deleteById($productId);
} catch (Exception $e) {
//continue if $productId not found
//TODO Logging
continue;
}
}
}
/**
* @group functional
*/
public function testSearchByDocument()
{
$client = $this->_getClient(array('persistent' => false));
$index = $client->getIndex('elastica_test');
$index->create(array('index' => array('number_of_shards' => 1, 'number_of_replicas' => 0)), true);
$type = new Type($index, 'mlt_test');
$type->addDocuments(array(new Document(1, array('visible' => true, 'name' => 'bruce wayne batman')), new Document(2, array('visible' => true, 'name' => 'bruce wayne')), new Document(3, array('visible' => false, 'name' => 'bruce wayne')), new Document(4, array('visible' => true, 'name' => 'batman')), new Document(5, array('visible' => false, 'name' => 'batman')), new Document(6, array('visible' => true, 'name' => 'superman')), new Document(7, array('visible' => true, 'name' => 'spiderman'))));
$index->refresh();
$doc = $type->getDocument(1);
// Return all similar from id
$mltQuery = new MoreLikeThis();
$mltQuery->setMinTermFrequency(1);
$mltQuery->setMinDocFrequency(1);
$mltQuery->setLike($doc);
$query = new Query($mltQuery);
$resultSet = $type->search($query);
$this->assertEquals(4, $resultSet->count());
$mltQuery = new MoreLikeThis();
$mltQuery->setMinTermFrequency(1);
$mltQuery->setMinDocFrequency(1);
$mltQuery->setLike($doc);
$query = new Query\BoolQuery();
$query->addMust($mltQuery);
$this->hideDeprecated();
// Return just the visible similar from id
$filter = new Query\BoolQuery();
$filterTerm = new Query\Term();
$filterTerm->setTerm('visible', true);
$filter->addMust($filterTerm);
$query->addFilter($filter);
$this->showDeprecated();
$resultSet = $type->search($query);
$this->assertEquals(2, $resultSet->count());
// Return all similar from source
$mltQuery = new MoreLikeThis();
$mltQuery->setMinTermFrequency(1);
$mltQuery->setMinDocFrequency(1);
$mltQuery->setMinimumShouldMatch(90);
$mltQuery->setLike($type->getDocument(1)->setId(''));
$query = new Query($mltQuery);
$resultSet = $type->search($query);
$this->assertEquals(1, $resultSet->count());
// Legacy test with filter
$mltQuery = new MoreLikeThis();
$mltQuery->setMinTermFrequency(1);
$mltQuery->setMinDocFrequency(1);
$mltQuery->setLike($doc);
$query = new Query\BoolQuery();
$query->addMust($mltQuery);
$this->hideDeprecated();
// Return just the visible similar
$filter = new BoolFilter();
$filterTerm = new Term();
$filterTerm->setTerm('visible', true);
$filter->addMust($filterTerm);
$query->addFilter($filter);
$this->showDeprecated();
$resultSet = $type->search($query);
$this->assertEquals(2, $resultSet->count());
}
/**
* Create new mappings or update existing mappings
*/
public function updateMappings()
{
/** @var ClassMetadata[] $metadatas */
$metadatas = $this->sm->getMetadataFactory()->getAllMetadata();
// Refresh all the mappings
foreach ($metadatas as $metadata) {
// if we're in the dev env, set the number of replicas to be 0
if ($this->env == 'dev' || $this->env == 'test_local') {
$metadata->numberOfReplicas = 0;
}
// create the index if it doesn't exist yet
$indexName = $metadata->timeSeriesScale ? $metadata->getCurrentTimeSeriesIndex() : $metadata->index;
/** @var Index $index */
$index = $this->client->getIndex($indexName);
if (!$index->exists()) {
$this->client->createIndex($indexName, $metadata->getSettings());
}
// create the type if it doesn't exist yet
if ($metadata->type) {
$type = new Type($index, $metadata->type);
if (!$type->exists()) {
$this->client->createType($metadata);
}
// update the mapping
$result = $this->client->updateMapping($metadata);
if (true !== $result) {
echo "Warning: Failed to update mapping for index '{$indexName}', type '{$metadata->type}'. Reason: {$result}\n";
}
}
}
}
/**
* @group functional
*/
public function testSearch()
{
$client = $this->_getClient();
$index = new Index($client, 'test');
$index->create(array(), true);
$index->getSettings()->setNumberOfReplicas(0);
//$index->getSettings()->setNumberOfShards(1);
$type = new Type($index, 'helloworldmlt');
$mapping = new Mapping($type, array('email' => array('store' => 'yes', 'type' => 'string', 'index' => 'analyzed'), 'content' => array('store' => 'yes', 'type' => 'string', 'index' => 'analyzed')));
$mapping->setSource(array('enabled' => false));
$type->setMapping($mapping);
$doc = new Document(1000, array('email' => '*****@*****.**', 'content' => 'This is a sample post. Hello World Fuzzy Like This!'));
$type->addDocument($doc);
$doc = new Document(1001, array('email' => '*****@*****.**', 'content' => 'This is a fake nospam email address for gmail'));
$type->addDocument($doc);
// Refresh index
$index->refresh();
$mltQuery = new MoreLikeThis();
$mltQuery->setLikeText('fake gmail sample');
$mltQuery->setFields(array('email', 'content'));
$mltQuery->setMaxQueryTerms(1);
$mltQuery->setMinDocFrequency(1);
$mltQuery->setMinTermFrequency(1);
$query = new Query();
$query->setFields(array('email', 'content'));
$query->setQuery($mltQuery);
$resultSet = $type->search($query);
$resultSet->getResponse()->getData();
$this->assertEquals(2, $resultSet->count());
}
public function testQuery()
{
$client = $this->_getClient();
$index = new Index($client, 'test');
$index->create(array(), true);
$type = new Type($index, 'constant_score');
$doc = new Document(1, array('id' => 1, 'email' => '*****@*****.**', 'username' => 'hans'));
$type->addDocument($doc);
$doc = new Document(2, array('id' => 2, 'email' => '*****@*****.**', 'username' => 'emil'));
$type->addDocument($doc);
$doc = new Document(3, array('id' => 3, 'email' => '*****@*****.**', 'username' => 'ruth'));
$type->addDocument($doc);
// Refresh index
$index->refresh();
$boost = 1.3;
$query_match = new MatchAll();
$query = new ConstantScore();
$query->setQuery($query_match);
$query->setBoost($boost);
$expectedArray = array('constant_score' => array('query' => $query_match->toArray(), 'boost' => $boost));
$this->assertEquals($expectedArray, $query->toArray());
$resultSet = $type->search($query);
$results = $resultSet->getResults();
$this->assertEquals($resultSet->count(), 3);
$this->assertEquals($results[1]->getScore(), 1);
}
/**
* @return void
*/
protected function setUp()
{
$this->type = $this->getMockType();
$this->index = $this->getMockIndex();
$this->client = $this->getMockClient();
// now that index is setup, we can use it for mocking the Type class method getIndex
$this->type->method('getIndex')->willReturn($this->index);
}
/**
* @param integer $id
* @param Type $type
*/
public function removeIndexById($id, Type $type)
{
try {
$type->deleteById($id);
} catch (\InvalidArgumentException $e) {
} catch (NotFoundException $e) {
}
}
/**
* @param string|\Elastica\Type $type
*
* @return $this
*/
public function setType($type)
{
if ($type instanceof Type) {
$this->setIndex($type->getIndex()->getName());
$type = $type->getName();
}
$this->_type = (string) $type;
return $this;
}
/**
* @param Type $type
* @param int $docs
*
* @return array
*/
private function _addDocs(Type $type, $docs)
{
$insert = array();
for ($i = 1; $i <= $docs; $i++) {
$insert[] = new Document($i, array('_id' => $i, 'key' => 'value'));
}
$type->addDocuments($insert);
$type->getIndex()->refresh();
return $insert;
}
public function testParentMapping()
{
$index = $this->_createIndex();
$parenttype = new Type($index, 'parenttype');
$parentmapping = new Mapping($parenttype, array('name' => array('type' => 'string', 'store' => 'yes')));
$parenttype->setMapping($parentmapping);
$childtype = new Type($index, 'childtype');
$childmapping = new Mapping($childtype, array('name' => array('type' => 'string', 'store' => 'yes')));
$childmapping->setParam('_parent', array('type' => 'parenttype'));
$childtype->setMapping($childmapping);
}
/**
* Insert the repository objects in the type index
*
* @param \Closure $loggerClosure A logging function
* @param array $options
*/
public function populate(\Closure $loggerClosure = null, array $options = array())
{
if ($loggerClosure) {
$loggerClosure('Indexing groups');
}
$groups = \Group::getQueryBuilder()->active()->getModels();
$documents = array();
foreach ($groups as $group) {
$documents[] = $this->transformer->transform($group, array());
}
$this->groupType->addDocuments($documents);
}
/**
* Insert the repository objects in the type index
*
* @param \Closure $loggerClosure A logging function
* @param array $options
*/
public function populate(\Closure $loggerClosure = null, array $options = array())
{
if ($loggerClosure) {
$loggerClosure('Indexing messages');
}
$messages = \Message::getQueryBuilder()->active()->getModels();
$documents = array();
foreach ($messages as $message) {
$documents[] = $this->transformer->transform($message);
}
$this->messageType->addDocuments($documents);
}
public function getMappingProperties(Type $type)
{
$array = array('name' => array('type' => 'multi_field', 'fields' => array('partial_name' => array('search_analyzer' => 'standard', 'index_analyzer' => 'shop', 'type' => 'string'), 'full_name' => array('type' => 'string'))), 'image' => array('type' => 'string', 'index' => 'no'));
switch ($type->getName()) {
case 'book':
$array += array('author' => array('type' => 'string'));
break;
case 'dvd':
$array += array('released' => array('type' => 'date'));
break;
}
return $array;
}
/**
* @group functional
*/
public function testSearch()
{
$index = $this->_createIndex();
$index->getSettings()->setNumberOfReplicas(0);
$type = new Type($index, 'helloworld');
$doc = new Document(1, array('email' => '*****@*****.**', 'username' => 'test 7/6 123', 'test' => array('2', '3', '5')));
$type->addDocument($doc);
// Refresh index
$index->refresh();
$queryString = new QueryString(Util::escapeTerm('test 7/6'));
$resultSet = $type->search($queryString);
$this->assertEquals(1, $resultSet->count());
}
/**
* @param string $typeName
* @return \Elastica\Type
* @throws \Exception
*/
protected function deleteMapping($typeName)
{
$type = new Elastica\Type($this->index, $typeName);
if ($type->exists()) {
try {
$type->delete();
} catch (\Exception $e) {
throw new \Exception('Could not delete type ' . $type->getName() . '');
}
}
$type = new Elastica\Type($this->index, $typeName);
return $type;
}
/**
* @group unit
*/
public function testToArrayFromReference()
{
$client = $this->_getClient();
$index = new Index($client, 'test');
$type = new Type($index, 'helloworld');
$field = 'image';
$query = new Image();
$query->setFieldFeature($field, 'CEDD');
$query->setFieldHash($field, 'BIT_SAMPLING');
$query->setFieldBoost($field, 100);
$query->setImageByReference($field, $index->getName(), $type->getName(), 10);
$jsonString = '{"image":{"image":{"feature":"CEDD","hash":"BIT_SAMPLING","boost":100,"index":"test","type":"helloworld","id":10,"path":"image"}}}';
$this->assertEquals($jsonString, json_encode($query->toArray()));
}
public function testSearch()
{
$client = $this->_getClient();
$index = new Index($client, 'test');
$index->create(array(), true);
$type = new Type($index, 'helloworld');
$doc = new Document(1, array('id' => 1, 'email' => '*****@*****.**', 'username' => 'hans', 'test' => array('2', '3', '5')));
$type->addDocument($doc);
$doc = new Document(2, array('id' => 2, 'email' => '*****@*****.**', 'username' => 'emil', 'test' => array('1', '3', '6')));
$type->addDocument($doc);
$doc = new Document(3, array('id' => 3, 'email' => '*****@*****.**', 'username' => 'ruth', 'test' => array('2', '3', '7')));
$type->addDocument($doc);
// Refresh index
$index->refresh();
$boolQuery = new Bool();
$termQuery1 = new Term(array('test' => '2'));
$boolQuery->addMust($termQuery1);
$resultSet = $type->search($boolQuery);
$this->assertEquals(2, $resultSet->count());
$termQuery2 = new Term(array('test' => '5'));
$boolQuery->addMust($termQuery2);
$resultSet = $type->search($boolQuery);
$this->assertEquals(1, $resultSet->count());
$termQuery3 = new Term(array('username' => 'hans'));
$boolQuery->addMust($termQuery3);
$resultSet = $type->search($boolQuery);
$this->assertEquals(1, $resultSet->count());
$termQuery4 = new Term(array('username' => 'emil'));
$boolQuery->addMust($termQuery4);
$resultSet = $type->search($boolQuery);
$this->assertEquals(0, $resultSet->count());
}
public function testSearch()
{
$client = $this->_getClient();
$index = new Index($client, 'test');
$index->create(array(), true);
$index->getSettings()->setNumberOfReplicas(0);
//$index->getSettings()->setNumberOfShards(1);
$type = new Type($index, 'helloworld');
$doc = new Document(1, array('email' => '*****@*****.**', 'username' => 'hanswurst', 'test' => array('2', '3', '5')));
$type->addDocument($doc);
// Refresh index
$index->refresh();
$queryString = new QueryString('test*');
$resultSet = $type->search($queryString);
$this->assertEquals(1, $resultSet->count());
}
public function testNegativeBoost()
{
$keyword = "vital";
$negativeKeyword = "mercury";
$query = new Boosting();
$positiveQuery = new \Elastica\Query\Term(array('name' => $keyword));
$negativeQuery = new \Elastica\Query\Term(array('name' => $negativeKeyword));
$query->setPositiveQuery($positiveQuery);
$query->setNegativeQuery($negativeQuery);
$query->setNegativeBoost(0.2);
$response = $this->type->search($query);
$results = $response->getResults();
$this->assertEquals($response->getTotalHits(), 4);
$lastResult = $results[3]->getData();
$this->assertEquals($lastResult['name'], $this->sampleData[2]['name']);
}
/**
* @param array $documentsForSave
*/
private function saveInfoInElastic(array $documentsForSave)
{
if (count($documentsForSave) == 0) {
return;
}
$resultUpdate = $this->elasticaType->addDocuments($documentsForSave);
if (!$resultUpdate->isOk()) {
$this->log->error('Ошибка записи документов');
exit;
}
}
private function deleteWarmers($warmers)
{
foreach (array_keys($warmers) as $name) {
$this->outputIndented("\tDeleting {$name}...");
$name = urlencode($name);
$path = "_warmer/{$name}";
$this->pageType->getIndex()->request($path, 'DELETE');
$this->output("done\n");
}
return Status::newGood();
}
public function testScriptScore()
{
$scriptString = "_score * doc['price'].value";
$script = new Script($scriptString);
$query = new FunctionScore();
$query->addScriptScoreFunction($script);
$expected = array('function_score' => array('functions' => array(array('script_score' => array('script' => $scriptString)))));
$this->assertEquals($expected, $query->toArray());
$response = $this->type->search($query);
$results = $response->getResults();
// the document the highest price should be scored highest
$result0 = $results[0]->getData();
$this->assertEquals("Miller's Field", $result0['name']);
}
public function testParentMapping()
{
$index = $this->_createIndex();
$parenttype = new Type($index, 'parenttype');
$parentmapping = new Mapping($parenttype, array('name' => array('type' => 'string', 'store' => 'yes')));
$parenttype->setMapping($parentmapping);
$childtype = new Type($index, 'childtype');
$childmapping = new Mapping($childtype, array('name' => array('type' => 'string', 'store' => 'yes')));
$childmapping->setParent('parenttype');
$childtype->setMapping($childmapping);
$data = $childmapping->toArray();
$this->assertEquals('parenttype', $data[$childtype->getName()]['_parent']['type']);
$index->delete();
}
public function testQuery()
{
$client = $this->_getClient();
$index = new Index($client, 'test');
$index->create(array(), true);
$type = new Type($index, 'multi_match');
$doc = new Document(1, array('id' => 1, 'name' => 'Rodolfo', 'last_name' => 'Moraes'));
$type->addDocument($doc);
// Refresh index
$index->refresh();
$multiMatch = new MultiMatch();
$query = new Query();
$multiMatch->setQuery('Rodolfo');
$multiMatch->setFields(array('name', 'last_name'));
$query->setQuery($multiMatch);
$resultSet = $index->search($query);
$this->assertEquals(1, $resultSet->count());
$multiMatch->setQuery('Moraes');
$multiMatch->setFields(array('name', 'last_name'));
$query->setQuery($multiMatch);
$resultSet = $index->search($query);
$this->assertEquals(1, $resultSet->count());
}
public function testSearch()
{
$client = $this->_getClient();
$index = new Index($client, 'test');
$index->create(array(), true);
$index->getSettings()->setNumberOfReplicas(0);
//$index->getSettings()->setNumberOfShards(1);
$type = new Type($index, 'helloworldfuzzy');
$mapping = new Mapping($type, array('email' => array('store' => 'yes', 'type' => 'string', 'index' => 'analyzed'), 'content' => array('store' => 'yes', 'type' => 'string', 'index' => 'analyzed')));
$mapping->setSource(array('enabled' => false));
$type->setMapping($mapping);
$doc = new Document(1000, array('email' => '*****@*****.**', 'content' => 'This is a sample post. Hello World Fuzzy Like This!'));
$type->addDocument($doc);
// Refresh index
$index->refresh();
$fltQuery = new FuzzyLikeThis();
$fltQuery->setLikeText("sample gmail");
$fltQuery->addFields(array("email", "content"));
$fltQuery->setMinSimilarity(0.3);
$fltQuery->setMaxQueryTerms(3);
$resultSet = $type->search($fltQuery);
$this->assertEquals(1, $resultSet->count());
}
/**
* @inheritDoc
*/
public function findAllIds()
{
$generator = function (ScanAndScroll $scanAndScroll) {
foreach ($scanAndScroll as $scrollId => $resultSet) {
foreach ($resultSet as $result) {
(yield $result->getId());
}
}
};
$query = new Query(new Query\MatchAll());
$query->setFields([]);
$search = $this->type->createSearch($query);
$scanAndScroll = new ScanAndScroll($search);
return $generator($scanAndScroll);
}
/**
* @param array $removeCities
*/
public function removeCities($removeCities)
{
foreach ($removeCities as $departmentId => $newCities) {
$document = $this->departmentType->getDocument($departmentId);
$data = $document->getData();
if (isset($data['citiesIds'])) {
$cities = $data['citiesIds'];
foreach ($newCities as $newCity) {
unset($cities[$newCity]);
}
$data['citiesIds'] = $cities;
$document->setData($data);
$this->departmentType->updateDocument($document);
}
}
}
protected function setUp()
{
$typeName = Cache::TYPE_NAME;
$this->type = $this->prophesize('\\Elastica\\Type');
$this->type->request(Argument::any(), Argument::cetera())->willReturn(true);
$this->type->getName()->willReturn($typeName);
$this->index = $this->prophesize('\\Elastica\\Index');
$this->index->getType($typeName)->willReturn($this->type->reveal());
$this->index->exists()->willReturn(true);
$nsDoc = new Document('DoctrineNamespaceCacheKey[]', [Cache::VALUE_FIELD => serialize($this->namespaceId)]);
$this->type->getIndex()->willReturn($this->index->reveal());
$this->type->getDocument("DoctrineNamespaceCacheKey[]")->willReturn($nsDoc);
$this->client = $this->prophesize('\\Elastica\\Client');
$this->client->getIndex($this->indexName)->willReturn($this->index->reveal());
$this->cache = new Cache($this->client->reveal(), ['index' => $this->indexName]);
}
/**
* @group functional
*/
public function testNoSource()
{
$index = $this->_createIndex();
$type = new Type($index, 'user');
// Adds 1 document to the index
$doc1 = new Document(1, array('username' => 'ruflin', 'test' => array('2', '3', '5')));
$type->addDocument($doc1);
// To update index
$index->refresh();
$query = Query::create('ruflin');
$resultSet = $type->search($query);
// Disable source
$query->setSource(false);
$resultSetNoSource = $type->search($query);
$this->assertEquals(1, $resultSet->count());
$this->assertEquals(1, $resultSetNoSource->count());
// Tests if no source is in response except id
$result = $resultSetNoSource->current();
$this->assertEquals(1, $result->getId());
$this->assertEmpty($result->getData());
// Tests if source is in response except id
$result = $resultSet->current();
$this->assertEquals(1, $result->getId());
$this->assertNotEmpty($result->getData());
}
protected function setUp()
{
/** @var ElasticaService elasticaService */
$elasticaService = $this->getContainer()->get("revinate_search.elasticsearch_service");
$this->elasticaClient = $elasticaService->getInstance();
$this->index = new \Elastica\Index($this->elasticaClient, View::INDEX_NAME);
if (!$this->index->exists()) {
$this->index->create(array("index.number_of_replicas" => "0", "index.number_of_shards" => "1"));
$this->type = new \Elastica\Type($this->index, View::INDEX_TYPE);
$mappingJson = json_decode(file_get_contents(__DIR__ . "/../data/es/mapping.json"), true);
$mapping = new \Elastica\Type\Mapping($this->type, $mappingJson['properties']);
$this->type->setMapping($mapping);
} else {
$this->type = new \Elastica\Type($this->index, View::INDEX_TYPE);
}
$this->timeSeriesIndex = new \Elastica\Index($this->elasticaClient, StatusLog::INDEX_NAME . self::$timeSeriesTestDateSuffix);
if (!$this->timeSeriesIndex->exists()) {
$this->timeSeriesIndex->create(array("index.number_of_replicas" => "0", "index.number_of_shards" => "1"));
$this->timeSeriesType = new \Elastica\Type($this->timeSeriesIndex, StatusLog::INDEX_TYPE);
$mappingJson = json_decode(file_get_contents(__DIR__ . "/../data/es/statusLogMapping.json"), true);
$mapping = new \Elastica\Type\Mapping($this->timeSeriesType, $mappingJson['properties']);
$this->timeSeriesType->setMapping($mapping);
} else {
$this->timeSeriesType = new \Elastica\Type($this->timeSeriesIndex, StatusLog::INDEX_TYPE);
}
}