/**
* add an object to the elasticsearch index
*
* @param $object
* @param $objectId
* @return string
*/
public function addObject($object, $objectId)
{
$objectId = sha1($objectId);
$params = array_replace($this->defaultParams, array('id' => $objectId, 'body' => $object));
$this->client->index($params);
return $objectId;
}
/**
* @param JobInterface $job
* @return string jobId
*/
public function create(JobInterface $job)
{
$job->validate();
$jobData = ['index' => $this->index->getIndexNameCurrent(), 'type' => 'jobs', 'id' => $job->getId(), 'body' => $this->fillEmptyKeys($job->getData())];
$response = null;
$i = 0;
while ($i < 5) {
try {
$response = $this->client->index($jobData);
break;
} catch (ServerErrorResponseException $e) {
// ES server error, try again
$this->log('error', 'Elastic server error response', ['attemptNo' => $i, 'jobId' => $job->getId(), 'exception' => $e]);
}
sleep(1 + intval(pow(2, $i) / 2));
$i++;
}
if (!isset($response['created'])) {
throw new ApplicationException("Unable to index job", null, ['job' => $jobData, 'elasticResponse' => $response]);
}
//@todo: remove sleep in next (major) release
sleep(1);
$i = 0;
while ($i < 5) {
$resJob = $this->get($job->getId());
if ($resJob != null) {
return $response['_id'];
}
sleep(1 + intval(pow(2, $i) / 2));
$i++;
}
throw new ApplicationException("Unable to find job in index after creation", null, ['job' => $job->getData(), 'elasticResponse' => $response]);
}
/**
* Index all products
*
* @return number
*/
public function indexAll()
{
$products = $this->database->table('product')->fetchAll();
foreach ($products as $product) {
$this->es->index(['index' => $this->indexName, 'type' => 'product', 'id' => $product['id'], 'body' => $product->toArray()]);
}
return count($products);
}
public function testIndexDocument2()
{
/** @var \Iwai\Elasticsearch\FutureData $future */
$future = $this->client->index(['index' => 'my_index', 'type' => 'my_type', 'id' => 'my_id2', 'body' => ['testField' => 'abc']]);
$future->then(function ($response) {
$this->assertEquals(1, $response['created']);
});
$response = $future->wait();
}
/**
* {@inheritdoc}
*/
public function set($user, $imageIdentifier, array $imageData)
{
$params = $this->prepareParams($imageIdentifier, $imageData);
try {
return (bool) $this->client->index($params);
} catch (Exception $e) {
trigger_error('Elasticsearch metadata indexing failed for image: ' . $imageIdentifier, E_USER_WARNING);
return false;
}
}
/**
* @param Model $model
*
* @throws DocumentMissingException
*/
public function addDocumentToIndex(Model $model)
{
if (!$model->exists) {
throw new DocumentMissingException('Document does not exist.');
}
$params = $this->getConfigurator()->getParams($model);
// Get our document body data.
$params['body'] = $this->getConfigurator()->getDocumentData($model);
$this->client->index($params);
}
/**
* @param SavableModelInterface $model
* @throws CouldNotPersistException
*/
public function persist(SavableModelInterface $model)
{
$params = ['index' => $this->index, 'type' => $this->type];
if ($model->getId()) {
$params['body'] = ['doc' => $model->toArray()];
$params['id'] = $model->getId();
$this->client->update($params);
$model->markAsStored();
return;
}
$params['body'] = $model->toArray();
$response = $this->client->index($params);
$model->id = $response['_id'];
$model->markAsStored();
}
/**
* {@inheritdoc}
*/
public function saveDocument(Searchable $model)
{
$document = $this->constructDocument($model);
if (!$this->indexIsNested($model->getSearchIndex())) {
$this->client->index($document);
return;
}
list($index, $type) = $this->retrieveNestedIndex($model->getSearchIndex());
$class = $this->retrieveParentClass($model->getSearchIndex());
$parent = $model->belongsTo($class, null, null, class_basename($class))->getResults();
$parentData = $this->client->get(['id' => $parent->getKey(), 'type' => $parent->getSearchType(), 'index' => $parent->getSearchIndex()])['_source'];
if (!isset($parentData[$type])) {
$parentData[$type] = [];
}
$children = Collection::make($parentData[$type]);
if ($child = $children->first(function ($child) use($model) {
return $child[$model->getKeyName()] == $model->getKey();
})) {
$newChildren = $children->map(function ($child) use($model) {
if ($child[$model->getKeyName()] == $model->getKey()) {
$child = $model->documentToArray();
if (!isset($document[$model->getKeyName()])) {
$child[$model->getKeyName()] = $model->getKey();
}
}
return $child;
});
} else {
$newChildren = $children->push($model->documentToArray());
}
$this->client->update(['id' => $parent->getKey(), 'type' => $parent->getSearchType(), 'index' => $parent->getSearchIndex(), 'body' => ['doc' => [$type => $newChildren]]]);
}
/**
* @iterations 1000
* @group large
*/
public function asyncLarge()
{
$asyncDoc = $this->largeDocument;
$asyncDoc['client']['future'] = 'lazy';
$response = $this->client->index($asyncDoc);
$response = $response['body']['created'];
}
public static function addDocuments(\ElasticSearch\Client $client, $num = 3, $tag = 'cool')
{
$options = array('refresh' => true);
while ($num-- > 0) {
$doc = array('title' => "One cool document {$tag}", 'rank' => rand(1, 10));
$client->index($doc, $num + 1, $options);
}
return $client;
}
/**
* Execute the request by elasticsearch client
*
* @param Client $client
* @return ResponseInterface
*/
public function executeByElasticClient(Client $client)
{
$responseClass = $this->getResponseClassOfRequest();
/** @var IndexResponseInterface $response */
$response = new $responseClass();
$rawResult = RawResponse::build($client->index($this->toElasticClient()));
$response = $response->build($rawResult);
$this->getDocument()->setId($response->id());
return $response;
}
/**
* @iterations 10
* @group large
*/
public function asyncLarge()
{
$responses = [];
$asyncDoc = $this->largeDocument;
$asyncDoc['client']['future'] = 'lazy';
for ($i = 0; $i < 1000; $i++) {
$responses[] = $this->client->index($asyncDoc);
}
$responses[999]->wait();
}
public function handleMessage(Change $change, Message $message, Channel $channel)
{
if (!$change->getProduct()) {
throw new \InvalidArgumentException("Badly routed message '{$message->content}' with routing key '{$message->routingKey}'.");
}
$product = $change->getProduct();
/** @var Eshop $eshop */
$eshop = $this->eshopRepository->getOneById($product->getEshopId());
$product->setEshop($eshop);
$categoryIds = $product->getCategoryIds();
if (!empty($categoryIds)) {
$product->setCategories($this->categoryRepository->find(["_id" => ['$in' => $product->getCategoryIds()]]));
}
if (!$this->elasticsearch->indices()->existsAlias(["name" => $this->catalogIndexAliasName])) {
$this->initIndex();
}
$response = $this->elasticsearch->index(["index" => $this->catalogIndexAliasName, "type" => ProductMeta::SHORT_NAME, "id" => (string) $product->getId(), "version" => $product->getV(), "version_type" => "external_gte", "body" => ProductMeta::toObject($product, "json:")]);
$this->log->info("Indexed Product#{$product->getId()} (v={$product->getV()}, created=" . json_encode($response["created"]) . ").");
$channel->ack($message);
}
/**
* Index a single document
*
* @param Posts $post
*/
protected function doIndex(Posts $post)
{
$params = [];
$karma = $post->number_views + ($post->votes_up - $post->votes_down) * 10 + $post->number_replies;
if ($karma > 0) {
$params['body'] = ['id' => $post->id, 'title' => $post->title, 'category' => $post->categories_id, 'content' => $post->content, 'karma' => $karma];
$params['index'] = $this->config->get('index', 'phosphorum');
$params['type'] = 'post';
$params['id'] = 'post-' . $post->id;
$this->client->index($params);
}
}
public function handle()
{
$params = array();
$params['hosts'] = array('http://10.0.2.2:9200');
$es = new Client($params);
$models = Winwin::all();
foreach ($models as $model) {
$es->index(['index' => 'winwins', 'type' => 'winwins', 'id' => $model->id, 'body' => $model->toArray()]);
}
Log::info('Bancame indexed');
$models = Group::all();
foreach ($models as $model) {
$es->index(['index' => 'winwins', 'type' => 'groups', 'id' => $model->id, 'body' => $model->toArray()]);
}
Log::info('Groups indexed');
$models = UserDetail::all();
foreach ($models as $model) {
$es->index(['index' => 'winwins', 'type' => 'users', 'id' => $model->id, 'body' => $model->toArray()]);
}
Log::info('Users indexed');
}
/**
* @param Searchable $type
* @param bool|true $needsLoading
*/
public function add(Searchable $type, $needsLoading = true)
{
//clone object so we do not touch original one.
//this was messing with translatable library.
//since the parent model saves before the translations,
//translations wouldn't be in database and we overwrite them here by loading those translations
$type = clone $type;
/*
* make sure the relations are initialised when creating a new object
* else searching might fail since some relations expect to be an array and it would be indexed as null
*/
if ($needsLoading) {
$type->load(array_keys($this->config->getWith($type->getSearchableType())));
}
$data = $this->data($type);
$this->client->index($data);
}
/**
* Inserts (a.k.a indexes) the document into the specified index.
*
* @param string $index The name of the index to insert the document.
* @param string $type The type (a.k.a collection) to insert the document.
* @param array $doc The document to insert.
* @param integer $id The Id to assign the document being inserted. If null Id is auto-assigned (recommended).
*
* @return array Like [
* '_shards' => [
* 'total' => (Integer),
* 'failed' => (Integer),
* 'successful' = > (Integer),
* ],
* '_index' => (String),
* '_type' => (String),
* '_id' => (String),
* '_version' => (Integer),
* 'created' => (Boolean),
* ];
*
* @throws UsageException When $index, $type, or $doc is null.
*/
public function insert($index, $type, array $doc, $id = null)
{
$errorMessages = [];
if (empty($index)) {
$errorMessages[] = '$index cannot be null.';
}
if (empty($type)) {
$errorMessages[] = '$type cannot be null.';
}
if (empty($doc)) {
$errorMessages[] = "An empty document cannot be inserted. That doesn't make any sense.";
}
if (!empty($errorMessages)) {
throw new UsageException(implode("\n", $errorMessages));
}
$req = ['index' => $index, 'type' => $type, 'body' => $doc];
if (!empty($id)) {
$req['id'] = $id;
}
return $this->client->index($req);
}
/**
* Saves a given entity or set of entities.
* Use the returned instance for further operations as
* the save operation might have changed the entity instance completely.
* @param ElasticsearchEntity|ElasticsearchEntity[] $param an entitiy or set of entities
* @return ElasticsearchEntity|ElasticsearchEntity[] the saved entity or entities
*/
public function save($param)
{
if (is_array($param)) {
foreach ($param as $p) {
if ($p instanceof ElasticsearchEntity) {
$this->save($p);
}
}
} else {
if ($param instanceof ElasticsearchEntity) {
$params = array('index' => $this->index, 'type' => $this->getType());
if ($param->getId()) {
$params['id'] = $param->getId();
}
$params['body'] = $param->getSource();
$result = $this->elasticsearchClient->index($params);
$this->updateEntityWithDocument($param, $result);
}
}
return $param;
}
protected function saveNode($title, $content, $url, $date)
{
$content = $this->cleanContent($content);
if (is_null($content)) {
$this->counter['empty_url']++;
return false;
}
$hashId = md5($url);
$params = ['index' => 'sites', 'type' => 'default', 'id' => $hashId, 'body' => ['title' => $title, 'content' => $content, 'hash_id' => $hashId, 'date' => $date ? Carbon::createFromTimestamp($date)->toW3cString() : null, 'url' => $url]];
try {
$response = $this->client->index($params);
if (!$response['created']) {
$this->counter['updated']++;
} else {
$this->counter['indexed']++;
}
} catch (ElasticsearchException $e) {
$this->error("# Cannot Index content of {$url}. Check \"failed:{$url}\" key in redis for more information");
$this->redis->hMSet("failed:{$url}", ['error' => $e->getMessage(), 'params' => $url]);
$this->counter['failed_index']++;
}
return true;
}
public function copyOne($from, $to, array $source = [], $parent = null)
{
$pathFrom = $from instanceof DocumentPath ? $from : new DocumentPath($from);
$pathTo = $to instanceof DocumentPath ? $to : new DocumentPath($to);
if (!$pathFrom->isValid() || !$pathTo->isValid()) {
throw new InvalidArgumentException('FROM or TO path is not valid');
}
// Get source
if (!$source) {
$params = ['index' => $pathFrom->getIndex(), 'type' => $pathFrom->getType(), 'id' => $pathFrom->getId(), 'ignore' => 404];
if ($parent) {
$params['parent'] = $parent;
}
$response = $this->clientFrom->get($params);
if (!is_array($response)) {
throw new InvalidArgumentException(sprintf('Failed to find %s', $pathFrom->getPath()));
}
$source = $response['_source'];
}
// Create
$params = ['index' => $pathTo->getIndex(), 'type' => $pathTo->getType(), 'id' => $pathTo->getId(), 'body' => $source];
if ($parent) {
$params['parent'] = $parent;
}
$created = false;
try {
$response = $this->clientTo->index($params);
$created = is_array($response) && ($response['created'] || $response['_version'] > 0);
} catch (BadRequest400Exception $e) {
// Tag may contain "/" in id and it's not good, because we can't index this document
// $created = true;
}
if (!$created) {
throw new Exception(sprintf('Failed to create %s', $pathTo->getPath()));
}
return $created;
}
/**
* @inheritdoc
*/
public function index(array $params = [])
{
return $this->client->index($params);
}
/**
* Maintain the search service info by passing the product to it
*
* @param Product $product
*/
public function maintainSearch(Product $product)
{
return $this->client->index(['body' => $product->toArray(), 'index' => self::INDEX, 'type' => self::TYPE, 'id' => $product->getId()]);
}
/**
* @iterations 100
* @group large
*/
public function guzzleLarge()
{
$this->guzzleClient->index($this->largeDocument);
}
/**
* {@inheritDoc}
*/
protected function write(array $record)
{
$this->client->index($record['formatted']);
}
/**
* @param string $identity
* @param Document $document
*/
public function upsert($identity, Document $document)
{
$this->client->index(['index' => $this->index, 'type' => $this->type, 'id' => $identity, 'body' => $document->serialize(), 'refresh' => true]);
}
/**
* Index a single document
*
* @param Client $client
* @param Posts $post
*/
protected function doIndex($client, $post)
{
$karma = $post->number_views + ($post->votes_up - $post->votes_down) * 10 + $post->number_replies;
if ($karma > 0) {
$params = array();
$params['body'] = array('id' => $post->id, 'title' => $post->title, 'category' => $post->categories_id, 'content' => $post->content, 'karma' => $karma);
$params['index'] = 'phosphorum';
$params['type'] = 'post';
$params['id'] = 'post-' . $post->id;
$ret = $client->index($params);
var_dump($ret);
}
}
function it_adds_a_searchable_object_to_the_search_index(Client $elasticsearch, Searchable $searchableObject)
{
$elasticsearch->index(['index' => $this->indexName, 'type' => $this->searchableType, 'id' => $this->searchableId, 'body' => $this->searchableBody])->shouldBeCalled();
$this->upsertToIndex($searchableObject);
}
/**
* @param Supply $supply
*/
public function indexSupply(Supply $supply)
{
$params = ['index' => ElasticSearch::INDEX, 'type' => 'supply', 'id' => $supply->getId(), 'body' => ['polishName' => $supply->getPolishName()]];
$this->client->index($params);
$this->client->indices()->refresh();
}
public function indexMarkdownDocument(FilePAth $path, ParsedMarkdownDocument $parsedMarkdownDocument)
{
$params = array('id' => $path->toAbsoluteUrlString(), 'index' => $this->index, 'type' => self::MARKDOWN_DOCUMENT_TYPE, 'body' => array('title' => $parsedMarkdownDocument->getTitle(), 'content' => $parsedMarkdownDocument->getSource(), 'linked_paths' => $parsedMarkdownDocument->getLinkedPaths()));
return $this->client->index($params);
}
public function insertUser($user)
{
$params = ['index' => INDEX_NAME, 'type' => USER_TYPE, 'id' => $user['user_id'], 'body' => ['username' => $user["name"], 'avatar_link' => $user["avatar_link"]]];
$client = new Client();
return $client->index($params);
}
