/** * Flushes the items in this Bulk object to Elasticsearch. If there are no items, this method does nothing. * * (In particular, note that if no items are in the Bulk object, the bulk flush event is not fired.) * * @throws PartialFailureException */ public function flush() { // We might have nothing to do if ($this->getItemCount() == 0) { return; } $this->dispatchEvent(Events::BULK_FLUSH, new BulkFlushEvent($this->index, $this->type, $this->getItemCount())); $params = ['index' => $this->index, 'type' => $this->type, 'body' => $this->itemList]; $result = $this->elasticSearch->bulk($params); if (count($result['items']) != $this->itemCount) { $expected = $this->itemCount; $actual = count($result['items']); throw new PartialFailureException("The wrong number of items was stored; expected {$expected}, but stored {$actual}"); } if (boolval($result['errors'])) { $errorList = []; foreach ($result['items'] as $item) { if ($item['index']['status'] < 200 || $item['index']['status'] >= 300) { $errorList[] = $item; } } throw new PartialFailureException("Some items failed. " . json_encode($errorList)); } $this->clear(); }
/** * {@inheritdoc} */ protected function setUp() { parent::setUp(); $this->client = ClientBuilder::create()->build(); $this->deleteIndex(); $this->client->indices()->create(array_filter(['index' => self::INDEX_NAME, 'mapping' => $this->getMapping()])); $bulkBody = []; foreach ($this->getDataArray() as $type => $documents) { foreach ($documents as $id => $document) { $bulkBody[] = ['index' => ['_index' => self::INDEX_NAME, '_type' => $type, '_id' => $id]]; $bulkBody[] = $document; } } $this->client->bulk(['body' => $bulkBody]); $this->client->indices()->refresh(); }
/** * Flushes the current query container to the index, used for bulk queries execution. */ public function commit() { $this->isReadOnly('Commit'); $this->bulkQueries = array_merge($this->bulkQueries, $this->bulkParams); $this->client->bulk($this->bulkQueries); $this->flush(); $this->bulkQueries = []; }
public function it_adds_multiple_searchable_objects_to_the_search_index(Client $elasticsearch, Searchable $searchableObject) { $searchableObjects = [$searchableObject]; $params = []; $params['body'][] = ['index' => ['_id' => $this->searchableId, '_index' => $this->indexName, '_type' => $this->searchableType]]; $params['body'][] = $this->searchableBody; $elasticsearch->bulk($params)->shouldBeCalled(); $this->upsertToIndex($searchableObjects); }
/** * Insert an array of documents. * * @param array $docs */ public function insertDocuments(array $docs) { $params = []; while (!empty($docs)) { $doc = array_shift($docs); $params['body'][] = ['index' => ['_index' => env('ES_INDEX'), '_type' => $this->type, '_id' => uniqid()]]; $params['body'][] = $doc; $results = $this->client->bulk($params); } }
/** * @param ShopIndex $index * @param int[] $ids */ public function index(ShopIndex $index, $ids) { if (empty($ids)) { return; } $blog = $this->provider->get($ids); $remove = array_diff($ids, array_keys($blog)); $documents = []; foreach ($blog as $row) { $documents[] = ['index' => ['_id' => $row->getId()]]; $documents[] = $row; } foreach ($remove as $id) { $documents[] = ['delete' => ['_id' => $id]]; } if (empty($documents)) { return; } $this->client->bulk(['index' => $index->getName(), 'type' => 'blog', 'body' => $documents]); }
private function copyData(Client $client, $source_index, $destination_index, $batch_size) { $search_params = array("search_type" => "scan", "scroll" => "30s", "size" => 50, "index" => $source_index, "body" => array("query" => array("match_all" => array()))); $search_responses = new SearchResponseIterator($client, $search_params); foreach ($search_responses as $search_response) { $index_params = array(); $index_params['index'] = $destination_index; foreach ($search_response['hits']['hits'] as $hit) { $index_params['body'][] = array('type' => $hit['_type'], 'id' => $hit['_id']); $index_params['body'][] = $hit['source']; } $client->bulk($index_params); } }
/** * Flushes the items in this Bulk object to Elasticsearch. If there are no items, this method does nothing. * * (In particular, note that if no items are in the Bulk object, the bulk flush event is not fired.) * * @throws PartialFailureException */ public function flush() { // We might have nothing to do if ($this->getItemCount() == 0) { return; } $this->dispatchEvent(Events::BULK_FLUSH, new BulkFlushEvent($this->index, $this->type, $this->getItemCount())); $params = ['index' => $this->index, 'type' => $this->type, 'body' => $this->itemList]; $result = $this->elasticSearch->bulk($params); if (!empty($result->errors)) { throw new PartialFailureException(count($result->errors) . " items failed.", $result->errors); } $this->itemList = []; $this->itemCount = 0; }
/** * @param string $type entity name * @param array $data [[id => body]] * @return array * * @see http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-bulk.html */ public function addToIndexBulk($type, array $data) { $queries = []; foreach ($data as $id => $body) { $queries[] = ['index' => ['_id' => $id]]; $queries[] = $body; } $res = parent::bulk(['index' => $this->index, 'type' => $type, 'body' => $queries]); foreach ($res['items'] as $queries) { foreach ($queries as $query) { if (isset($query['error'])) { throw new ElasticsearchException($query['error']); } } } return $res; }
/** * @param ShopIndex $index * @param string[] $numbers * @return \string[] */ public function indexProducts(ShopIndex $index, $numbers) { if (empty($numbers)) { return; } $products = $this->provider->get($index->getShop(), $numbers); $remove = array_diff($numbers, array_keys($products)); $documents = []; foreach ($products as $product) { $documents[] = ['index' => ['_id' => $product->getNumber()]]; $documents[] = $product; } foreach ($remove as $number) { $documents[] = ['delete' => ['_id' => $number]]; } $this->client->bulk(['index' => $index->getName(), 'type' => ProductMapping::TYPE, 'body' => $documents]); }
/** * {@inheritDoc} */ public function executeBulk(BulkRequestInterface $bulk) { $bulkParams = ['body' => $bulk->getOperations()]; $rawBulkResponse = $this->client->bulk($bulkParams); /** * @var BulkResponseInterface */ $bulkResponse = $this->objectManager->create('Smile\\ElasticsuiteCore\\Api\\Index\\Bulk\\BulkResponseInterface', ['rawResponse' => $rawBulkResponse]); if ($bulkResponse->hasErrors()) { foreach ($bulkResponse->aggregateErrorsByReason() as $error) { $sampleDocumentIds = implode(', ', array_slice($error['document_ids'], 0, 10)); $errorMessages = [sprintf("Bulk %s operation failed %d times in index %s for type %s", $error['operation'], $error['count'], $error['index'], $error['document_type']), sprintf("Error (%s) : %s", $error['error']['type'], $error['error']['reason']), sprintf("Failed doc ids sample : %s", $sampleDocumentIds)]; $this->logger->error(implode(" ", $errorMessages)); } } return $bulkResponse; }
/** * @param ShopIndex $index * @param int[] $groupIds */ public function indexProperties(ShopIndex $index, $groupIds) { if (empty($groupIds)) { return; } /** @var Group[] $properties */ $properties = $this->provider->get($index->getShop(), $groupIds); $remove = array_diff($groupIds, array_keys($properties)); $documents = []; foreach ($properties as $property) { $documents[] = ['index' => ['_id' => $property->getId()]]; $documents[] = $property; } foreach ($remove as $id) { $documents[] = ['delete' => ['_id' => $id]]; } $this->client->bulk(['index' => $index->getName(), 'type' => PropertyMapping::TYPE, 'body' => $documents]); }
/** * Inserts the current query container to the index, used for bulk queries execution. * * @param array $params Parameters that will be passed to the flush or refresh queries. * * @return null|array */ public function commit(array $params = []) { $this->isReadOnly('Commit'); if (!empty($this->bulkQueries)) { $bulkQueries = array_merge($this->bulkQueries, $this->bulkParams); $this->bulkQueries = []; $bulkResponse = $this->client->bulk($bulkQueries); switch ($this->getCommitMode()) { case 'flush': $this->flush($params); break; case 'refresh': default: $this->refresh($params); break; } return $bulkResponse; } return null; }
/** * @param CsvFile $file * @param LoadOptions $options * @param $primaryIndex * @return bool */ public function loadFile(CsvFile $file, LoadOptions $options, $primaryIndex = null) { $csvHeader = $file->getHeader(); $params = ['body' => []]; $iBulk = 1; foreach ($file as $i => $line) { // skip header if (!$i) { continue; } $lineData = array_combine($csvHeader, $line); if ($primaryIndex) { if (!array_key_exists($primaryIndex, $lineData)) { $this->logger->error(sprintf("CSV error: Missing id column %s on line %s", $primaryIndex, $i + 1)); return false; } $params['body'][] = ['index' => ['_index' => $options->getIndex(), '_type' => $options->getType(), '_id' => $lineData[$primaryIndex]]]; } else { $params['body'][] = ['index' => ['_index' => $options->getIndex(), '_type' => $options->getType()]]; } $params['body'][] = $lineData; if ($i % $options->getBulkSize() == 0) { $this->logger->info(sprintf("Write %s batch %d to %s start", $options->getType(), $iBulk, $options->getIndex())); $responses = $this->client->bulk($params); $this->logger->info(sprintf("Write %s batch %d to %s took %d ms", $options->getType(), $iBulk, $options->getIndex(), $responses['took'])); $params = ['body' => []]; if ($responses['errors'] !== false) { if (!empty($responses['items'])) { foreach ($responses['items'] as $itemResult) { if (!empty($itemResult['index']['error'])) { if (is_array($itemResult['index']['error'])) { $this->logger->error(sprintf("ES error: %s", $this->getErrorMessageFromErrorField($itemResult['index']['error']))); } else { $this->logger->error(sprintf("ES error: %s", $itemResult['index']['error'])); } return false; } } } return false; } $iBulk++; unset($responses); } } if (!empty($params['body'])) { $this->logger->info(sprintf("Write %s batch %d to %s start", $options->getType(), $iBulk, $options->getIndex())); $responses = $this->client->bulk($params); $this->logger->info(sprintf("Write %s batch %d to %s took %d ms", $options->getType(), $iBulk, $options->getIndex(), $responses['took'])); if ($responses['errors'] !== false) { if (!empty($responses['items'])) { foreach ($responses['items'] as $itemResult) { if (!empty($itemResult['index']['error'])) { if (is_array($itemResult['index']['error'])) { $this->logger->error(sprintf("ES error: %s", $this->getErrorMessageFromErrorField($itemResult['index']['error']))); } else { $this->logger->error(sprintf("ES error: %s", $itemResult['index']['error'])); } return false; } } } return false; } unset($responses); } return true; }
<?php include __DIR__ . '/../../vendor/autoload.php'; use Utils\ZillowClient; use Elasticsearch\Client; $client = new Client(['hosts' => ['http://localhost:9200']]); $zClient = new ZillowClient(); $results = json_decode($zClient->getDataForAllZips()); $params = []; $params['index'] = 'hrqls'; $params['type'] = 'houseData'; foreach ($results as $item) { $params['body'][] = array('create' => array('_id' => sha1($item->location->lat . $item->location->lon))); $params['body'][] = array('state' => $item->state, 'city' => $item->city, 'zip' => $item->zip, 'location' => array('lat' => $item->location->lat, 'lon' => $item->location->lon), 'avgHomeValueIndex' => $item->homeData->avgHomeValueIndex, 'avgHomesRecentlySold' => $item->homeData->avgHomesRecentlySold, 'avgPropertyTax' => $item->homeData->avgPropertyTax, 'turnoverWithinLastYear' => $item->homeData->turnoverWithinLastYear); } $client->bulk($params);
/** * {@inheritdoc} */ public function handleBatch(array $records) { $this->client->bulk($this->getFormatter()->formatBatch($records)); }
private function flushBulk() { $this->client->bulk($this->currentBulk); $this->currentBulkSize = 0; $this->currentBulk = ['body' => []]; }
/** * @inheritdoc */ public function bulk(array $params = []) { return $this->client->bulk($params); }
/** * Execute a bulk statement on index;. * * @param $params * * @return array */ public function bulkStatement(array $params) { return $this->elastic->bulk($params); }