/** * @param string $typeName * @return \Elastica\Type * @throws \Exception */ protected function deleteMapping($typeName) { $type = new Elastica\Type($this->index, $typeName); if ($type->exists()) { try { $type->delete(); } catch (\Exception $e) { throw new \Exception('Could not delete type ' . $type->getName() . ''); } } $type = new Elastica\Type($this->index, $typeName); return $type; }
public function getMappingProperties(Type $type) { $array = array('name' => array('type' => 'multi_field', 'fields' => array('partial_name' => array('search_analyzer' => 'standard', 'index_analyzer' => 'shop', 'type' => 'string'), 'full_name' => array('type' => 'string'))), 'image' => array('type' => 'string', 'index' => 'no')); switch ($type->getName()) { case 'book': $array += array('author' => array('type' => 'string')); break; case 'dvd': $array += array('released' => array('type' => 'date')); break; } return $array; }
/** * @group unit */ public function testToArrayFromReference() { $client = $this->_getClient(); $index = new Index($client, 'test'); $type = new Type($index, 'helloworld'); $field = 'image'; $query = new Image(); $query->setFieldFeature($field, 'CEDD'); $query->setFieldHash($field, 'BIT_SAMPLING'); $query->setFieldBoost($field, 100); $query->setImageByReference($field, $index->getName(), $type->getName(), 10); $jsonString = '{"image":{"image":{"feature":"CEDD","hash":"BIT_SAMPLING","boost":100,"index":"test","type":"helloworld","id":10,"path":"image"}}}'; $this->assertEquals($jsonString, json_encode($query->toArray())); }
protected function setUp() { $typeName = Cache::TYPE_NAME; $this->type = $this->prophesize('\\Elastica\\Type'); $this->type->request(Argument::any(), Argument::cetera())->willReturn(true); $this->type->getName()->willReturn($typeName); $this->index = $this->prophesize('\\Elastica\\Index'); $this->index->getType($typeName)->willReturn($this->type->reveal()); $this->index->exists()->willReturn(true); $nsDoc = new Document('DoctrineNamespaceCacheKey[]', [Cache::VALUE_FIELD => serialize($this->namespaceId)]); $this->type->getIndex()->willReturn($this->index->reveal()); $this->type->getDocument("DoctrineNamespaceCacheKey[]")->willReturn($nsDoc); $this->client = $this->prophesize('\\Elastica\\Client'); $this->client->getIndex($this->indexName)->willReturn($this->index->reveal()); $this->cache = new Cache($this->client->reveal(), ['index' => $this->indexName]); }
private function updateWarmers($warmers) { $type = $this->pageType->getName(); foreach ($warmers as $name => $contents) { // The types field comes back on warmers but it can't be sent back in $this->outputIndented("\tUpdating {$name}..."); $name = urlencode($name); $path = "{$type}/_warmer/{$name}"; try { $this->pageType->getIndex()->request($path, 'PUT', $contents); } catch (ResponseException $e) { if (preg_match('/dynamic scripting for \\[.*\\] disabled/', $e->getResponse()->getError())) { $this->output("couldn't create dynamic script!\n"); return Status::newFatal(new RawMessage("Couldn't create the dynamic script required for Cirrus to work properly. " . "For now, Cirrus requires dynamic scripting. It'll switch to sandboxed Groovy when it " . "updates to support Elasticsearch 1.3.1 we promise. For now enable dynamic scripting and " . "keep Elasticsearch safely not accessible to people you don't trust. You should always " . "do that, but especially when dynamic scripting is enabled.")); } } $this->output("done\n"); } return Status::newGood(); }
/** * @group functional */ public function testParentMapping() { $index = $this->_createIndex(); $childtype = new Type($index, 'childtype'); $childmapping = new Mapping($childtype, array('name' => array('type' => 'string', 'store' => true))); $childmapping->setParent('parenttype'); $childtype->setMapping($childmapping); $data = $childmapping->toArray(); $this->assertEquals('parenttype', $data[$childtype->getName()]['_parent']['type']); $parenttype = new Type($index, 'parenttype'); $parentmapping = new Mapping($parenttype, array('name' => array('type' => 'string', 'store' => true))); $parenttype->setMapping($parentmapping); }
private function reindexInternal(Type $type, Type $oldType, $children, $childNumber, $chunkSize, $retryAttempts) { $filter = null; $messagePrefix = ""; if ($childNumber === 1 && $children === 1) { $this->outputIndented("\t\tStarting single process reindex\n"); } else { if ($childNumber >= $children) { $this->error("Invalid parameters - childNumber >= children ({$childNumber} >= {$children}) ", 1); } $messagePrefix = "\t\t[{$childNumber}] "; $this->outputIndented($messagePrefix . "Starting child process reindex\n"); // Note that it is not ok to abs(_uid.hashCode) because hashCode(Integer.MIN_VALUE) == Integer.MIN_VALUE $filter = new \CirrusSearch\Extra\Filter\IdHashMod($children, $childNumber); } $properties = $this->mappingConfig[$oldType->getName()]['properties']; try { $query = new Query(); $query->setFields(array('_id', '_source')); if ($filter) { $query->setQuery(new \Elastica\Query\Filtered(new \Elastica\Query\MatchAll(), $filter)); } // Note here we dump from the current index (using the alias) so we can use Connection::getPageType $result = $oldType->search($query, array('search_type' => 'scan', 'scroll' => '1h', 'size' => $chunkSize)); $totalDocsToReindex = $result->getResponse()->getData(); $totalDocsToReindex = $totalDocsToReindex['hits']['total']; $this->outputIndented($messagePrefix . "About to reindex {$totalDocsToReindex} documents\n"); $operationStartTime = microtime(true); $completed = 0; $self = $this; Util::iterateOverScroll($this->index, $result->getResponse()->getScrollId(), '1h', function ($results) use($properties, $retryAttempts, $messagePrefix, $self, $type, &$completed, $totalDocsToReindex, $operationStartTime) { $documents = array(); foreach ($results as $result) { $documents[] = $self->buildNewDocument($result, $properties); } $self->withRetry($retryAttempts, $messagePrefix, 'retrying as singles', function () use($self, $type, $messagePrefix, $documents) { $self->sendDocuments($type, $messagePrefix, $documents); }); $completed += sizeof($results); $rate = round($completed / (microtime(true) - $operationStartTime)); $this->outputIndented($messagePrefix . "Reindexed {$completed}/{$totalDocsToReindex} documents at {$rate}/second\n"); }, 0, $retryAttempts, function ($e, $errors) use($self, $messagePrefix) { $self->sleepOnRetry($e, $errors, $messagePrefix, 'fetching documents to reindex'); }); $this->outputIndented($messagePrefix . "All done\n"); } catch (ExceptionInterface $e) { // Note that we can't fail the master here, we have to check how many documents are in the new index in the master. $type = get_class($e); $message = ElasticsearchIntermediary::extractMessage($e); LoggerFactory::getInstance('CirrusSearch')->warning("Search backend error during reindex. Error type is '{type}' and message is: {message}", array('type' => $type, 'message' => $message)); die(1); } }
/** * @param array $document * @param string $type * @throws \Exception */ protected function addDocumentToElastic(array $document, $type = 'issue') { switch ($type) { case 'issue': if (isset($document['custom_fields'])) { foreach ($document['custom_fields'] as $customField) { switch ($customField['id']) { case 4: $document['typo3_version'] = isset($customField['value']) ? $customField['value'] : '-'; break; case 5: $document['php_version'] = isset($customField['value']) ? $customField['value'] : '-'; break; case 8: $document['complexity'] = isset($customField['value']) ? $customField['value'] : '-'; break; case 15: $document['isregression'] = isset($customField['value']) ? true : false; break; case 18: $document['focus']['name'] = isset($customField['value']) ? $customField['value'] : '-'; break; default: } } } $document['updated_on'] = $this->fixDateFormat($document['updated_on']); $document['created_on'] = $this->fixDateFormat($document['created_on']); $type = new Elastica\Type($this->elasticIndex, 'issue'); try { $testIfDocExists = $type->getDocument($document['id']); } catch (\Exception $e) { $message = new \WMDB\Forger\Utilities\Slack\Message(); $message->sendMessage($document); } break; case 'review': $type = new Elastica\Type($this->elasticIndex, 'review'); break; case 'user': $type = new Elastica\Type($this->elasticIndex, 'user'); break; default: } #\TYPO3\Flow\var_dump($type); $doc = new Elastica\Document($document['id'], $document); #\TYPO3\Flow\var_dump($doc); $type->addDocument($doc); GeneralUtility::writeLine('+' . $type->getName() . ':' . $document['id'] . ' ', 'green', false); #sleep(1); }