/**
* @param string $typeName
* @return \Elastica\Type
* @throws \Exception
*/
protected function deleteMapping($typeName)
{
$type = new Elastica\Type($this->index, $typeName);
if ($type->exists()) {
try {
$type->delete();
} catch (\Exception $e) {
throw new \Exception('Could not delete type ' . $type->getName() . '');
}
}
$type = new Elastica\Type($this->index, $typeName);
return $type;
}
public function getMappingProperties(Type $type)
{
$array = array('name' => array('type' => 'multi_field', 'fields' => array('partial_name' => array('search_analyzer' => 'standard', 'index_analyzer' => 'shop', 'type' => 'string'), 'full_name' => array('type' => 'string'))), 'image' => array('type' => 'string', 'index' => 'no'));
switch ($type->getName()) {
case 'book':
$array += array('author' => array('type' => 'string'));
break;
case 'dvd':
$array += array('released' => array('type' => 'date'));
break;
}
return $array;
}
/**
* @group unit
*/
public function testToArrayFromReference()
{
$client = $this->_getClient();
$index = new Index($client, 'test');
$type = new Type($index, 'helloworld');
$field = 'image';
$query = new Image();
$query->setFieldFeature($field, 'CEDD');
$query->setFieldHash($field, 'BIT_SAMPLING');
$query->setFieldBoost($field, 100);
$query->setImageByReference($field, $index->getName(), $type->getName(), 10);
$jsonString = '{"image":{"image":{"feature":"CEDD","hash":"BIT_SAMPLING","boost":100,"index":"test","type":"helloworld","id":10,"path":"image"}}}';
$this->assertEquals($jsonString, json_encode($query->toArray()));
}
protected function setUp()
{
$typeName = Cache::TYPE_NAME;
$this->type = $this->prophesize('\\Elastica\\Type');
$this->type->request(Argument::any(), Argument::cetera())->willReturn(true);
$this->type->getName()->willReturn($typeName);
$this->index = $this->prophesize('\\Elastica\\Index');
$this->index->getType($typeName)->willReturn($this->type->reveal());
$this->index->exists()->willReturn(true);
$nsDoc = new Document('DoctrineNamespaceCacheKey[]', [Cache::VALUE_FIELD => serialize($this->namespaceId)]);
$this->type->getIndex()->willReturn($this->index->reveal());
$this->type->getDocument("DoctrineNamespaceCacheKey[]")->willReturn($nsDoc);
$this->client = $this->prophesize('\\Elastica\\Client');
$this->client->getIndex($this->indexName)->willReturn($this->index->reveal());
$this->cache = new Cache($this->client->reveal(), ['index' => $this->indexName]);
}
private function updateWarmers($warmers)
{
$type = $this->pageType->getName();
foreach ($warmers as $name => $contents) {
// The types field comes back on warmers but it can't be sent back in
$this->outputIndented("\tUpdating {$name}...");
$name = urlencode($name);
$path = "{$type}/_warmer/{$name}";
try {
$this->pageType->getIndex()->request($path, 'PUT', $contents);
} catch (ResponseException $e) {
if (preg_match('/dynamic scripting for \\[.*\\] disabled/', $e->getResponse()->getError())) {
$this->output("couldn't create dynamic script!\n");
return Status::newFatal(new RawMessage("Couldn't create the dynamic script required for Cirrus to work properly. " . "For now, Cirrus requires dynamic scripting. It'll switch to sandboxed Groovy when it " . "updates to support Elasticsearch 1.3.1 we promise. For now enable dynamic scripting and " . "keep Elasticsearch safely not accessible to people you don't trust. You should always " . "do that, but especially when dynamic scripting is enabled."));
}
}
$this->output("done\n");
}
return Status::newGood();
}
/**
* @group functional
*/
public function testParentMapping()
{
$index = $this->_createIndex();
$childtype = new Type($index, 'childtype');
$childmapping = new Mapping($childtype, array('name' => array('type' => 'string', 'store' => true)));
$childmapping->setParent('parenttype');
$childtype->setMapping($childmapping);
$data = $childmapping->toArray();
$this->assertEquals('parenttype', $data[$childtype->getName()]['_parent']['type']);
$parenttype = new Type($index, 'parenttype');
$parentmapping = new Mapping($parenttype, array('name' => array('type' => 'string', 'store' => true)));
$parenttype->setMapping($parentmapping);
}
private function reindexInternal(Type $type, Type $oldType, $children, $childNumber, $chunkSize, $retryAttempts)
{
$filter = null;
$messagePrefix = "";
if ($childNumber === 1 && $children === 1) {
$this->outputIndented("\t\tStarting single process reindex\n");
} else {
if ($childNumber >= $children) {
$this->error("Invalid parameters - childNumber >= children ({$childNumber} >= {$children}) ", 1);
}
$messagePrefix = "\t\t[{$childNumber}] ";
$this->outputIndented($messagePrefix . "Starting child process reindex\n");
// Note that it is not ok to abs(_uid.hashCode) because hashCode(Integer.MIN_VALUE) == Integer.MIN_VALUE
$filter = new \CirrusSearch\Extra\Filter\IdHashMod($children, $childNumber);
}
$properties = $this->mappingConfig[$oldType->getName()]['properties'];
try {
$query = new Query();
$query->setFields(array('_id', '_source'));
if ($filter) {
$query->setQuery(new \Elastica\Query\Filtered(new \Elastica\Query\MatchAll(), $filter));
}
// Note here we dump from the current index (using the alias) so we can use Connection::getPageType
$result = $oldType->search($query, array('search_type' => 'scan', 'scroll' => '1h', 'size' => $chunkSize));
$totalDocsToReindex = $result->getResponse()->getData();
$totalDocsToReindex = $totalDocsToReindex['hits']['total'];
$this->outputIndented($messagePrefix . "About to reindex {$totalDocsToReindex} documents\n");
$operationStartTime = microtime(true);
$completed = 0;
$self = $this;
Util::iterateOverScroll($this->index, $result->getResponse()->getScrollId(), '1h', function ($results) use($properties, $retryAttempts, $messagePrefix, $self, $type, &$completed, $totalDocsToReindex, $operationStartTime) {
$documents = array();
foreach ($results as $result) {
$documents[] = $self->buildNewDocument($result, $properties);
}
$self->withRetry($retryAttempts, $messagePrefix, 'retrying as singles', function () use($self, $type, $messagePrefix, $documents) {
$self->sendDocuments($type, $messagePrefix, $documents);
});
$completed += sizeof($results);
$rate = round($completed / (microtime(true) - $operationStartTime));
$this->outputIndented($messagePrefix . "Reindexed {$completed}/{$totalDocsToReindex} documents at {$rate}/second\n");
}, 0, $retryAttempts, function ($e, $errors) use($self, $messagePrefix) {
$self->sleepOnRetry($e, $errors, $messagePrefix, 'fetching documents to reindex');
});
$this->outputIndented($messagePrefix . "All done\n");
} catch (ExceptionInterface $e) {
// Note that we can't fail the master here, we have to check how many documents are in the new index in the master.
$type = get_class($e);
$message = ElasticsearchIntermediary::extractMessage($e);
LoggerFactory::getInstance('CirrusSearch')->warning("Search backend error during reindex. Error type is '{type}' and message is: {message}", array('type' => $type, 'message' => $message));
die(1);
}
}
/**
* @param array $document
* @param string $type
* @throws \Exception
*/
protected function addDocumentToElastic(array $document, $type = 'issue')
{
switch ($type) {
case 'issue':
if (isset($document['custom_fields'])) {
foreach ($document['custom_fields'] as $customField) {
switch ($customField['id']) {
case 4:
$document['typo3_version'] = isset($customField['value']) ? $customField['value'] : '-';
break;
case 5:
$document['php_version'] = isset($customField['value']) ? $customField['value'] : '-';
break;
case 8:
$document['complexity'] = isset($customField['value']) ? $customField['value'] : '-';
break;
case 15:
$document['isregression'] = isset($customField['value']) ? true : false;
break;
case 18:
$document['focus']['name'] = isset($customField['value']) ? $customField['value'] : '-';
break;
default:
}
}
}
$document['updated_on'] = $this->fixDateFormat($document['updated_on']);
$document['created_on'] = $this->fixDateFormat($document['created_on']);
$type = new Elastica\Type($this->elasticIndex, 'issue');
try {
$testIfDocExists = $type->getDocument($document['id']);
} catch (\Exception $e) {
$message = new \WMDB\Forger\Utilities\Slack\Message();
$message->sendMessage($document);
}
break;
case 'review':
$type = new Elastica\Type($this->elasticIndex, 'review');
break;
case 'user':
$type = new Elastica\Type($this->elasticIndex, 'user');
break;
default:
}
#\TYPO3\Flow\var_dump($type);
$doc = new Elastica\Document($document['id'], $document);
#\TYPO3\Flow\var_dump($doc);
$type->addDocument($doc);
GeneralUtility::writeLine('+' . $type->getName() . ':' . $document['id'] . ' ', 'green', false);
#sleep(1);
}
