Uses _bulk to send documents to the server.
|
public addDocuments ( array $docs ) : |
||
| $docs | array | Array of Elastica\Document |
| return | ||
/**
* Insert the repository objects in the type index
*
* @param \Closure $loggerClosure A logging function
* @param array $options
*/
public function populate(\Closure $loggerClosure = null, array $options = array())
{
if ($loggerClosure) {
$loggerClosure('Indexing messages');
}
$messages = \Message::getQueryBuilder()->active()->getModels();
$documents = array();
foreach ($messages as $message) {
$documents[] = $this->transformer->transform($message);
}
$this->messageType->addDocuments($documents);
}
/**
* Insert the repository objects in the type index
*
* @param \Closure $loggerClosure A logging function
* @param array $options
*/
public function populate(\Closure $loggerClosure = null, array $options = array())
{
if ($loggerClosure) {
$loggerClosure('Indexing groups');
}
$groups = \Group::getQueryBuilder()->active()->getModels();
$documents = array();
foreach ($groups as $group) {
$documents[] = $this->transformer->transform($group, array());
}
$this->groupType->addDocuments($documents);
}
/**
* @group functional
*/
public function testSearchByDocument()
{
$client = $this->_getClient(array('persistent' => false));
$index = $client->getIndex('elastica_test');
$index->create(array('index' => array('number_of_shards' => 1, 'number_of_replicas' => 0)), true);
$type = new Type($index, 'mlt_test');
$type->addDocuments(array(new Document(1, array('visible' => true, 'name' => 'bruce wayne batman')), new Document(2, array('visible' => true, 'name' => 'bruce wayne')), new Document(3, array('visible' => false, 'name' => 'bruce wayne')), new Document(4, array('visible' => true, 'name' => 'batman')), new Document(5, array('visible' => false, 'name' => 'batman')), new Document(6, array('visible' => true, 'name' => 'superman')), new Document(7, array('visible' => true, 'name' => 'spiderman'))));
$index->refresh();
$doc = $type->getDocument(1);
// Return all similar from id
$mltQuery = new MoreLikeThis();
$mltQuery->setMinTermFrequency(1);
$mltQuery->setMinDocFrequency(1);
$mltQuery->setLike($doc);
$query = new Query($mltQuery);
$resultSet = $type->search($query);
$this->assertEquals(4, $resultSet->count());
$mltQuery = new MoreLikeThis();
$mltQuery->setMinTermFrequency(1);
$mltQuery->setMinDocFrequency(1);
$mltQuery->setLike($doc);
$query = new Query\BoolQuery();
$query->addMust($mltQuery);
$this->hideDeprecated();
// Return just the visible similar from id
$filter = new Query\BoolQuery();
$filterTerm = new Query\Term();
$filterTerm->setTerm('visible', true);
$filter->addMust($filterTerm);
$query->addFilter($filter);
$this->showDeprecated();
$resultSet = $type->search($query);
$this->assertEquals(2, $resultSet->count());
// Return all similar from source
$mltQuery = new MoreLikeThis();
$mltQuery->setMinTermFrequency(1);
$mltQuery->setMinDocFrequency(1);
$mltQuery->setMinimumShouldMatch(90);
$mltQuery->setLike($type->getDocument(1)->setId(''));
$query = new Query($mltQuery);
$resultSet = $type->search($query);
$this->assertEquals(1, $resultSet->count());
// Legacy test with filter
$mltQuery = new MoreLikeThis();
$mltQuery->setMinTermFrequency(1);
$mltQuery->setMinDocFrequency(1);
$mltQuery->setLike($doc);
$query = new Query\BoolQuery();
$query->addMust($mltQuery);
$this->hideDeprecated();
// Return just the visible similar
$filter = new BoolFilter();
$filterTerm = new Term();
$filterTerm->setTerm('visible', true);
$filter->addMust($filterTerm);
$query->addFilter($filter);
$this->showDeprecated();
$resultSet = $type->search($query);
$this->assertEquals(2, $resultSet->count());
}
/**
* @param Type $type
* @param int $docs
*
* @return array
*/
private function _addDocs(Type $type, $docs)
{
$insert = array();
for ($i = 1; $i <= $docs; $i++) {
$insert[] = new Document($i, array('_id' => $i, 'key' => 'value'));
}
$type->addDocuments($insert);
$type->getIndex()->refresh();
return $insert;
}
/**
* @param array $documentsForSave
*/
private function saveInfoInElastic(array $documentsForSave)
{
if (count($documentsForSave) == 0) {
return;
}
$resultUpdate = $this->elasticaType->addDocuments($documentsForSave);
if (!$resultUpdate->isOk()) {
$this->log->error('Ошибка записи документов');
exit;
}
}
/**
* This is really private.
*/
public function sendDocuments(Type $type, $messagePrefix, $documents)
{
try {
$type->addDocuments($documents);
} catch (ExceptionInterface $e) {
$errorType = get_class($e);
$message = ElasticsearchIntermediary::extractMessage($e);
$this->outputIndented($messagePrefix . "Error adding documents in bulk. Retrying as singles. Error type is '{$errorType}' and message is: {$message}");
foreach ($documents as $document) {
// Continue using the bulk api because we're used to it.
$type->addDocuments(array($document));
}
}
}
/**
* @group functional
*/
public function testOptimize()
{
$index = $this->_createIndex();
$type = new Type($index, 'optimize');
$docs = array();
$docs[] = new Document(1, array('foo' => 'bar'));
$docs[] = new Document(2, array('foo' => 'bar'));
$type->addDocuments($docs);
$index->refresh();
$stats = $index->getStats()->getData();
$this->assertEquals(0, $stats['_all']['primaries']['docs']['deleted']);
$type->deleteById(1);
$index->refresh();
$stats = $index->getStats()->getData();
$this->assertEquals(1, $stats['_all']['primaries']['docs']['deleted']);
$index->optimize(array('max_num_segments' => 1));
$stats = $index->getStats()->getData();
$this->assertEquals(0, $stats['_all']['primaries']['docs']['deleted']);
}
/**
* @group functional
*/
public function testSearchSetAnalyzer()
{
$client = $this->_getClient();
$index = new Index($client, 'test');
$index->create(array('analysis' => array('analyzer' => array('searchAnalyzer' => array('type' => 'custom', 'tokenizer' => 'standard', 'filter' => array('myStopWords'))), 'filter' => array('myStopWords' => array('type' => 'stop', 'stopwords' => array('The'))))), true);
$index->getSettings()->setNumberOfReplicas(0);
//$index->getSettings()->setNumberOfShards(1);
$type = new Type($index, 'helloworldfuzzy');
$mapping = new Mapping($type, array('email' => array('store' => 'yes', 'type' => 'string', 'index' => 'analyzed'), 'content' => array('store' => 'yes', 'type' => 'string', 'index' => 'analyzed')));
$mapping->setSource(array('enabled' => false));
$type->setMapping($mapping);
$type->addDocuments(array(new Document(1000, array('email' => '*****@*****.**', 'content' => 'The Fuzzy Test!')), new Document(1001, array('email' => '*****@*****.**', 'content' => 'Elastica Fuzzy Test'))));
// Refresh index
$index->refresh();
$fltQuery = new FuzzyLikeThis();
$fltQuery->addFields(array('email', 'content'));
$fltQuery->setLikeText('The');
$fltQuery->setMinSimilarity(0.1);
$fltQuery->setMaxQueryTerms(3);
// Test before analyzer applied, should return 1 result
$resultSet = $type->search($fltQuery);
$this->assertEquals(1, $resultSet->count());
$fltQuery->setParam('analyzer', 'searchAnalyzer');
$resultSet = $type->search($fltQuery);
$this->assertEquals(0, $resultSet->count());
}
/**
* @group functional
*/
public function testDeleteDocument()
{
$index = $this->_createIndex();
$type = new Type($index, 'user');
// Adds hans, john and rolf to the index
$docs = array(new Document(1, array('username' => 'hans', 'test' => array('2', '3', '5'))), new Document(2, array('username' => 'john', 'test' => array('1', '3', '6'))), new Document(3, array('username' => 'rolf', 'test' => array('2', '3', '7'))));
$type->addDocuments($docs);
$index->refresh();
$document = $type->getDocument(1);
$this->assertEquals(1, $document->getId());
$this->assertEquals('hans', $document->get('username'));
$this->assertEquals(3, $type->count());
$type->deleteDocument($document);
$index->refresh();
try {
$type->getDocument(1);
$this->fail('Document was not deleted');
} catch (NotFoundException $e) {
$this->assertTrue(true);
$this->assertEquals(2, $type->count(), 'Documents count in type should be 2');
}
}
/**
* @group functional
*/
public function testMoreLikeThisApi()
{
$client = $this->_getClient(array('persistent' => false));
$index = $client->getIndex('elastica_test');
$index->create(array('index' => array('number_of_shards' => 1, 'number_of_replicas' => 0)), true);
$type = new Type($index, 'mlt_test');
$type->addDocuments(array(new Document(1, array('visible' => true, 'name' => 'bruce wayne batman')), new Document(2, array('visible' => true, 'name' => 'bruce wayne')), new Document(3, array('visible' => false, 'name' => 'bruce wayne')), new Document(4, array('visible' => true, 'name' => 'batman')), new Document(5, array('visible' => false, 'name' => 'batman')), new Document(6, array('visible' => true, 'name' => 'superman')), new Document(7, array('visible' => true, 'name' => 'spiderman'))));
$index->refresh();
$document = $type->getDocument(1);
// Return all similar
$resultSet = $type->moreLikeThis($document, array('min_term_freq' => '1', 'min_doc_freq' => '1'));
$this->assertEquals(4, $resultSet->count());
// Return just the visible similar
$query = new Query();
$filterTerm = new Term();
$filterTerm->setTerm('visible', true);
$query->setPostFilter($filterTerm);
$resultSet = $type->moreLikeThis($document, array('min_term_freq' => '1', 'min_doc_freq' => '1'), $query);
$this->assertEquals(2, $resultSet->count());
}
/**
* @Testing count
*/
public function testCount()
{
$index = $this->_createIndex();
$type = new Type($index, 'user');
// Adds a list of documents with _bulk upload to the index
$docs = array();
$docs[] = new Document(2, array('username' => 'rolf', 'test' => array('1', '3', '6')));
$docs[] = new Document(3, array('username' => 'rolf', 'test' => array('2', '3', '7')));
$type->addDocuments($docs);
$index->refresh();
$resultSet = $type->search('rolf');
$this->assertEquals(2, $resultSet->count());
$count = $type->count('rolf');
$this->assertEquals(2, $count);
$resultSet = $type->count('rolf', true);
$this->assertEquals(2, $resultSet->count());
// Test if source is returned
$result = $resultSet->current();
$data = $result->getData();
$this->assertEquals('rolf', $data['username']);
}
/**
* @param $index
* @param $key
* @param $typ
* @param $type
* @param $options
* @param $trans
* @param $output
* @param $reset
* @param $showProgress
*/
protected function populateType($index, $key, Type $typ, $type, $options, $trans, $output, $reset, $showProgress)
{
if ($reset & $type) {
$this->resetter->reset($index, $type, $key, $output);
}
$output->writeln("Populating " . $key);
$this->jsonLdSerializer->getJsonLdFrameLoader()->setEsIndex($index);
$class = $this->configManager->getIndexConfiguration($index)->getType($key)->getType();
$size = $this->getSize($class);
// no object in triplestore
if (!current($size)) {
return;
}
$size = current($size)['count'];
$progress = $this->displayInitialAvancement($size, $options, $showProgress, $output);
$done = 0;
while ($done < $size) {
$resources = $this->getResources($class, $options, $done);
$docs = array();
/* @var Resource $add */
foreach ($resources as $resource) {
$types = $resource->all('rdf:type');
$mostAccurateType = $this->getMostAccurateType($types, $resource, $output, $class);
// index only the current resource if the most accurate type is the key populating
if ($class === $mostAccurateType) {
$doc = $trans->transform($resource->getUri(), $index, $mostAccurateType);
if ($doc) {
$docs[] = $doc;
}
}
}
// send documents to elasticsearch
if (count($docs)) {
$typ->addDocuments($docs);
} else {
$output->writeln("");
$output->writeln("nothing to index");
}
$done = $this->displayAvancement($options, $done, $size, $showProgress, $output, $progress);
//flushing manager for mem usage
$this->resourceManager->flush();
}
$progress->finish();
}
