public function testWithRetry() { $calls = 0; $func = function () use(&$calls) { $calls++; if ($calls <= 5) { throw new InvalidException(); } }; $self = $this; $errorCallbackCalls = 0; Util::withRetry(5, $func, function ($e, $errCount) use($self, &$errorCallbackCalls) { $errorCallbackCalls++; $self->assertEquals("Elastica\\Exception\\InvalidException", get_class($e)); }); $this->assertEquals(6, $calls); $this->assertEquals(5, $errorCallbackCalls); }
/** * @param int $attempts * @param string $messagePrefix * @param string $description * @param callable $func * @return mixed */ public function withRetry($attempts, $messagePrefix, $description, $func) { $self = $this; return Util::withRetry($attempts, $func, function ($e, $errors) use($self, $messagePrefix, $description) { $self->sleepOnRetry($e, $errors, $messagePrefix, $description); }); }
private function indexData() { $query = new Query(); $query->setFields(array('_id', '_type', '_source')); // Exclude content fields to save bandwidth $query->setSource(array('exclude' => array('text', 'source_text', 'opening_text', 'auxiliary_text'))); $query->setQuery(new Elastica\Query\Filtered(new Elastica\Query\MatchAll(), new Elastica\Filter\BoolAnd(array(new Elastica\Filter\Type(Connection::PAGE_TYPE_NAME), new Elastica\Filter\Term(array("namespace" => NS_MAIN)))))); $scrollOptions = array('search_type' => 'scan', 'scroll' => "15m", 'size' => $this->indexChunkSize); // TODO: only content index for now ( we'll have to check how it works with commons ) $sourceIndex = $this->getConnection()->getIndex($this->indexBaseName, Connection::CONTENT_INDEX_TYPE); $result = $sourceIndex->search($query, $scrollOptions); $totalDocsInIndex = $result->getResponse()->getData(); $totalDocsInIndex = $totalDocsInIndex['hits']['total']; $totalDocsToDump = $totalDocsInIndex; $scoreMethodName = $this->getOption('scoringMethod', 'quality'); $this->scoreMethod = SuggestScoringMethodFactory::getScoringMethod($scoreMethodName, $totalDocsInIndex); $builder = new SuggestBuilder($this->scoreMethod, $this->withGeo); $docsDumped = 0; $this->output("Indexing {$totalDocsToDump} documents ({$totalDocsInIndex} in the index)\n"); $self = $this; $destinationType = $this->getIndex()->getType(Connection::TITLE_SUGGEST_TYPE_NAME); $retryAttempts = $this->indexRetryAttempts; Util::iterateOverScroll($sourceIndex, $result->getResponse()->getScrollId(), '15m', function ($results) use($self, &$docsDumped, $totalDocsToDump, $builder, $destinationType, $retryAttempts) { $suggestDocs = array(); foreach ($results as $result) { $docsDumped++; $suggests = $builder->build($result->getId(), $result->getSource()); foreach ($suggests as $suggest) { $suggestDocs[] = new \Elastica\Document(null, $suggest); } } $self->outputProgress($docsDumped, $totalDocsToDump); Util::withRetry($retryAttempts, function () use($destinationType, $suggestDocs) { $destinationType->addDocuments($suggestDocs); }); }, 0, $retryAttempts); $this->output("Indexing done.\n"); }