public function testWithRetry()
{
$calls = 0;
$func = function () use(&$calls) {
$calls++;
if ($calls <= 5) {
throw new InvalidException();
}
};
$self = $this;
$errorCallbackCalls = 0;
Util::withRetry(5, $func, function ($e, $errCount) use($self, &$errorCallbackCalls) {
$errorCallbackCalls++;
$self->assertEquals("Elastica\\Exception\\InvalidException", get_class($e));
});
$this->assertEquals(6, $calls);
$this->assertEquals(5, $errorCallbackCalls);
}
/**
* @param int $attempts
* @param string $messagePrefix
* @param string $description
* @param callable $func
* @return mixed
*/
public function withRetry($attempts, $messagePrefix, $description, $func)
{
$self = $this;
return Util::withRetry($attempts, $func, function ($e, $errors) use($self, $messagePrefix, $description) {
$self->sleepOnRetry($e, $errors, $messagePrefix, $description);
});
}
private function indexData()
{
$query = new Query();
$query->setFields(array('_id', '_type', '_source'));
// Exclude content fields to save bandwidth
$query->setSource(array('exclude' => array('text', 'source_text', 'opening_text', 'auxiliary_text')));
$query->setQuery(new Elastica\Query\Filtered(new Elastica\Query\MatchAll(), new Elastica\Filter\BoolAnd(array(new Elastica\Filter\Type(Connection::PAGE_TYPE_NAME), new Elastica\Filter\Term(array("namespace" => NS_MAIN))))));
$scrollOptions = array('search_type' => 'scan', 'scroll' => "15m", 'size' => $this->indexChunkSize);
// TODO: only content index for now ( we'll have to check how it works with commons )
$sourceIndex = $this->getConnection()->getIndex($this->indexBaseName, Connection::CONTENT_INDEX_TYPE);
$result = $sourceIndex->search($query, $scrollOptions);
$totalDocsInIndex = $result->getResponse()->getData();
$totalDocsInIndex = $totalDocsInIndex['hits']['total'];
$totalDocsToDump = $totalDocsInIndex;
$scoreMethodName = $this->getOption('scoringMethod', 'quality');
$this->scoreMethod = SuggestScoringMethodFactory::getScoringMethod($scoreMethodName, $totalDocsInIndex);
$builder = new SuggestBuilder($this->scoreMethod, $this->withGeo);
$docsDumped = 0;
$this->output("Indexing {$totalDocsToDump} documents ({$totalDocsInIndex} in the index)\n");
$self = $this;
$destinationType = $this->getIndex()->getType(Connection::TITLE_SUGGEST_TYPE_NAME);
$retryAttempts = $this->indexRetryAttempts;
Util::iterateOverScroll($sourceIndex, $result->getResponse()->getScrollId(), '15m', function ($results) use($self, &$docsDumped, $totalDocsToDump, $builder, $destinationType, $retryAttempts) {
$suggestDocs = array();
foreach ($results as $result) {
$docsDumped++;
$suggests = $builder->build($result->getId(), $result->getSource());
foreach ($suggests as $suggest) {
$suggestDocs[] = new \Elastica\Document(null, $suggest);
}
}
$self->outputProgress($docsDumped, $totalDocsToDump);
Util::withRetry($retryAttempts, function () use($destinationType, $suggestDocs) {
$destinationType->addDocuments($suggestDocs);
});
}, 0, $retryAttempts);
$this->output("Indexing done.\n");
}
