/**
* Flushes the items in this Bulk object to Elasticsearch. If there are no items, this method does nothing.
*
* (In particular, note that if no items are in the Bulk object, the bulk flush event is not fired.)
*
* @throws PartialFailureException
*/
public function flush()
{
// We might have nothing to do
if ($this->getItemCount() == 0) {
return;
}
$this->dispatchEvent(Events::BULK_FLUSH, new BulkFlushEvent($this->index, $this->type, $this->getItemCount()));
$params = ['index' => $this->index, 'type' => $this->type, 'body' => $this->itemList];
$result = $this->elasticSearch->bulk($params);
if (count($result['items']) != $this->itemCount) {
$expected = $this->itemCount;
$actual = count($result['items']);
throw new PartialFailureException("The wrong number of items was stored; expected {$expected}, but stored {$actual}");
}
if (boolval($result['errors'])) {
$errorList = [];
foreach ($result['items'] as $item) {
if ($item['index']['status'] < 200 || $item['index']['status'] >= 300) {
$errorList[] = $item;
}
}
throw new PartialFailureException("Some items failed. " . json_encode($errorList));
}
$this->clear();
}
/**
* {@inheritdoc}
*/
protected function setUp()
{
parent::setUp();
$this->client = ClientBuilder::create()->build();
$this->deleteIndex();
$this->client->indices()->create(array_filter(['index' => self::INDEX_NAME, 'mapping' => $this->getMapping()]));
$bulkBody = [];
foreach ($this->getDataArray() as $type => $documents) {
foreach ($documents as $id => $document) {
$bulkBody[] = ['index' => ['_index' => self::INDEX_NAME, '_type' => $type, '_id' => $id]];
$bulkBody[] = $document;
}
}
$this->client->bulk(['body' => $bulkBody]);
$this->client->indices()->refresh();
}
/**
* Flushes the current query container to the index, used for bulk queries execution.
*/
public function commit()
{
$this->isReadOnly('Commit');
$this->bulkQueries = array_merge($this->bulkQueries, $this->bulkParams);
$this->client->bulk($this->bulkQueries);
$this->flush();
$this->bulkQueries = [];
}
public function it_adds_multiple_searchable_objects_to_the_search_index(Client $elasticsearch, Searchable $searchableObject)
{
$searchableObjects = [$searchableObject];
$params = [];
$params['body'][] = ['index' => ['_id' => $this->searchableId, '_index' => $this->indexName, '_type' => $this->searchableType]];
$params['body'][] = $this->searchableBody;
$elasticsearch->bulk($params)->shouldBeCalled();
$this->upsertToIndex($searchableObjects);
}
/**
* Insert an array of documents.
*
* @param array $docs
*/
public function insertDocuments(array $docs)
{
$params = [];
while (!empty($docs)) {
$doc = array_shift($docs);
$params['body'][] = ['index' => ['_index' => env('ES_INDEX'), '_type' => $this->type, '_id' => uniqid()]];
$params['body'][] = $doc;
$results = $this->client->bulk($params);
}
}
/**
* @param ShopIndex $index
* @param int[] $ids
*/
public function index(ShopIndex $index, $ids)
{
if (empty($ids)) {
return;
}
$blog = $this->provider->get($ids);
$remove = array_diff($ids, array_keys($blog));
$documents = [];
foreach ($blog as $row) {
$documents[] = ['index' => ['_id' => $row->getId()]];
$documents[] = $row;
}
foreach ($remove as $id) {
$documents[] = ['delete' => ['_id' => $id]];
}
if (empty($documents)) {
return;
}
$this->client->bulk(['index' => $index->getName(), 'type' => 'blog', 'body' => $documents]);
}
private function copyData(Client $client, $source_index, $destination_index, $batch_size)
{
$search_params = array("search_type" => "scan", "scroll" => "30s", "size" => 50, "index" => $source_index, "body" => array("query" => array("match_all" => array())));
$search_responses = new SearchResponseIterator($client, $search_params);
foreach ($search_responses as $search_response) {
$index_params = array();
$index_params['index'] = $destination_index;
foreach ($search_response['hits']['hits'] as $hit) {
$index_params['body'][] = array('type' => $hit['_type'], 'id' => $hit['_id']);
$index_params['body'][] = $hit['source'];
}
$client->bulk($index_params);
}
}
/**
* Flushes the items in this Bulk object to Elasticsearch. If there are no items, this method does nothing.
*
* (In particular, note that if no items are in the Bulk object, the bulk flush event is not fired.)
*
* @throws PartialFailureException
*/
public function flush()
{
// We might have nothing to do
if ($this->getItemCount() == 0) {
return;
}
$this->dispatchEvent(Events::BULK_FLUSH, new BulkFlushEvent($this->index, $this->type, $this->getItemCount()));
$params = ['index' => $this->index, 'type' => $this->type, 'body' => $this->itemList];
$result = $this->elasticSearch->bulk($params);
if (!empty($result->errors)) {
throw new PartialFailureException(count($result->errors) . " items failed.", $result->errors);
}
$this->itemList = [];
$this->itemCount = 0;
}
/**
* @param string $type entity name
* @param array $data [[id => body]]
* @return array
*
* @see http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-bulk.html
*/
public function addToIndexBulk($type, array $data)
{
$queries = [];
foreach ($data as $id => $body) {
$queries[] = ['index' => ['_id' => $id]];
$queries[] = $body;
}
$res = parent::bulk(['index' => $this->index, 'type' => $type, 'body' => $queries]);
foreach ($res['items'] as $queries) {
foreach ($queries as $query) {
if (isset($query['error'])) {
throw new ElasticsearchException($query['error']);
}
}
}
return $res;
}
/**
* @param ShopIndex $index
* @param string[] $numbers
* @return \string[]
*/
public function indexProducts(ShopIndex $index, $numbers)
{
if (empty($numbers)) {
return;
}
$products = $this->provider->get($index->getShop(), $numbers);
$remove = array_diff($numbers, array_keys($products));
$documents = [];
foreach ($products as $product) {
$documents[] = ['index' => ['_id' => $product->getNumber()]];
$documents[] = $product;
}
foreach ($remove as $number) {
$documents[] = ['delete' => ['_id' => $number]];
}
$this->client->bulk(['index' => $index->getName(), 'type' => ProductMapping::TYPE, 'body' => $documents]);
}
/**
* {@inheritDoc}
*/
public function executeBulk(BulkRequestInterface $bulk)
{
$bulkParams = ['body' => $bulk->getOperations()];
$rawBulkResponse = $this->client->bulk($bulkParams);
/**
* @var BulkResponseInterface
*/
$bulkResponse = $this->objectManager->create('Smile\\ElasticsuiteCore\\Api\\Index\\Bulk\\BulkResponseInterface', ['rawResponse' => $rawBulkResponse]);
if ($bulkResponse->hasErrors()) {
foreach ($bulkResponse->aggregateErrorsByReason() as $error) {
$sampleDocumentIds = implode(', ', array_slice($error['document_ids'], 0, 10));
$errorMessages = [sprintf("Bulk %s operation failed %d times in index %s for type %s", $error['operation'], $error['count'], $error['index'], $error['document_type']), sprintf("Error (%s) : %s", $error['error']['type'], $error['error']['reason']), sprintf("Failed doc ids sample : %s", $sampleDocumentIds)];
$this->logger->error(implode(" ", $errorMessages));
}
}
return $bulkResponse;
}
/**
* @param ShopIndex $index
* @param int[] $groupIds
*/
public function indexProperties(ShopIndex $index, $groupIds)
{
if (empty($groupIds)) {
return;
}
/** @var Group[] $properties */
$properties = $this->provider->get($index->getShop(), $groupIds);
$remove = array_diff($groupIds, array_keys($properties));
$documents = [];
foreach ($properties as $property) {
$documents[] = ['index' => ['_id' => $property->getId()]];
$documents[] = $property;
}
foreach ($remove as $id) {
$documents[] = ['delete' => ['_id' => $id]];
}
$this->client->bulk(['index' => $index->getName(), 'type' => PropertyMapping::TYPE, 'body' => $documents]);
}
/**
* Inserts the current query container to the index, used for bulk queries execution.
*
* @param array $params Parameters that will be passed to the flush or refresh queries.
*
* @return null|array
*/
public function commit(array $params = [])
{
$this->isReadOnly('Commit');
if (!empty($this->bulkQueries)) {
$bulkQueries = array_merge($this->bulkQueries, $this->bulkParams);
$this->bulkQueries = [];
$bulkResponse = $this->client->bulk($bulkQueries);
switch ($this->getCommitMode()) {
case 'flush':
$this->flush($params);
break;
case 'refresh':
default:
$this->refresh($params);
break;
}
return $bulkResponse;
}
return null;
}
/**
* @param CsvFile $file
* @param LoadOptions $options
* @param $primaryIndex
* @return bool
*/
public function loadFile(CsvFile $file, LoadOptions $options, $primaryIndex = null)
{
$csvHeader = $file->getHeader();
$params = ['body' => []];
$iBulk = 1;
foreach ($file as $i => $line) {
// skip header
if (!$i) {
continue;
}
$lineData = array_combine($csvHeader, $line);
if ($primaryIndex) {
if (!array_key_exists($primaryIndex, $lineData)) {
$this->logger->error(sprintf("CSV error: Missing id column %s on line %s", $primaryIndex, $i + 1));
return false;
}
$params['body'][] = ['index' => ['_index' => $options->getIndex(), '_type' => $options->getType(), '_id' => $lineData[$primaryIndex]]];
} else {
$params['body'][] = ['index' => ['_index' => $options->getIndex(), '_type' => $options->getType()]];
}
$params['body'][] = $lineData;
if ($i % $options->getBulkSize() == 0) {
$this->logger->info(sprintf("Write %s batch %d to %s start", $options->getType(), $iBulk, $options->getIndex()));
$responses = $this->client->bulk($params);
$this->logger->info(sprintf("Write %s batch %d to %s took %d ms", $options->getType(), $iBulk, $options->getIndex(), $responses['took']));
$params = ['body' => []];
if ($responses['errors'] !== false) {
if (!empty($responses['items'])) {
foreach ($responses['items'] as $itemResult) {
if (!empty($itemResult['index']['error'])) {
if (is_array($itemResult['index']['error'])) {
$this->logger->error(sprintf("ES error: %s", $this->getErrorMessageFromErrorField($itemResult['index']['error'])));
} else {
$this->logger->error(sprintf("ES error: %s", $itemResult['index']['error']));
}
return false;
}
}
}
return false;
}
$iBulk++;
unset($responses);
}
}
if (!empty($params['body'])) {
$this->logger->info(sprintf("Write %s batch %d to %s start", $options->getType(), $iBulk, $options->getIndex()));
$responses = $this->client->bulk($params);
$this->logger->info(sprintf("Write %s batch %d to %s took %d ms", $options->getType(), $iBulk, $options->getIndex(), $responses['took']));
if ($responses['errors'] !== false) {
if (!empty($responses['items'])) {
foreach ($responses['items'] as $itemResult) {
if (!empty($itemResult['index']['error'])) {
if (is_array($itemResult['index']['error'])) {
$this->logger->error(sprintf("ES error: %s", $this->getErrorMessageFromErrorField($itemResult['index']['error'])));
} else {
$this->logger->error(sprintf("ES error: %s", $itemResult['index']['error']));
}
return false;
}
}
}
return false;
}
unset($responses);
}
return true;
}
<?php
include __DIR__ . '/../../vendor/autoload.php';
use Utils\ZillowClient;
use Elasticsearch\Client;
$client = new Client(['hosts' => ['http://localhost:9200']]);
$zClient = new ZillowClient();
$results = json_decode($zClient->getDataForAllZips());
$params = [];
$params['index'] = 'hrqls';
$params['type'] = 'houseData';
foreach ($results as $item) {
$params['body'][] = array('create' => array('_id' => sha1($item->location->lat . $item->location->lon)));
$params['body'][] = array('state' => $item->state, 'city' => $item->city, 'zip' => $item->zip, 'location' => array('lat' => $item->location->lat, 'lon' => $item->location->lon), 'avgHomeValueIndex' => $item->homeData->avgHomeValueIndex, 'avgHomesRecentlySold' => $item->homeData->avgHomesRecentlySold, 'avgPropertyTax' => $item->homeData->avgPropertyTax, 'turnoverWithinLastYear' => $item->homeData->turnoverWithinLastYear);
}
$client->bulk($params);
/**
* {@inheritdoc}
*/
public function handleBatch(array $records)
{
$this->client->bulk($this->getFormatter()->formatBatch($records));
}
private function flushBulk()
{
$this->client->bulk($this->currentBulk);
$this->currentBulkSize = 0;
$this->currentBulk = ['body' => []];
}
/**
* @inheritdoc
*/
public function bulk(array $params = [])
{
return $this->client->bulk($params);
}
/**
* Execute a bulk statement on index;.
*
* @param $params
*
* @return array
*/
public function bulkStatement(array $params)
{
return $this->elastic->bulk($params);
}
