/**
* Flushes the items in this Bulk object to Elasticsearch. If there are no items, this method does nothing.
*
* (In particular, note that if no items are in the Bulk object, the bulk flush event is not fired.)
*
* @throws PartialFailureException
*/
public function flush()
{
// We might have nothing to do
if ($this->getItemCount() == 0) {
return;
}
$this->dispatchEvent(Events::BULK_FLUSH, new BulkFlushEvent($this->index, $this->type, $this->getItemCount()));
$params = ['index' => $this->index, 'type' => $this->type, 'body' => $this->itemList];
$result = $this->elasticSearch->bulk($params);
if (count($result['items']) != $this->itemCount) {
$expected = $this->itemCount;
$actual = count($result['items']);
throw new PartialFailureException("The wrong number of items was stored; expected {$expected}, but stored {$actual}");
}
if (boolval($result['errors'])) {
$errorList = [];
foreach ($result['items'] as $item) {
if ($item['index']['status'] < 200 || $item['index']['status'] >= 300) {
$errorList[] = $item;
}
}
throw new PartialFailureException("Some items failed. " . json_encode($errorList));
}
$this->clear();
}
public function main(Request $req, Application $app)
{
$client = new Client();
$params = array('index' => 'hrqls', 'type' => 'houseData', 'body' => ['from' => 0, 'size' => 100, 'filter' => array('range' => array('avgHomeValueIndex' => array('gte' => 0))), 'query' => ['match' => ['state' => 'Virginia']]]);
$results = $client->search($params)['hits']['hits'];
$responseObject = [];
$averageHouseValue = array('total' => 0, 'number' => 0);
$averageTurnover = array('total' => 0, 'number' => 0);
$maxHouseValue = 0;
$minHouseValue = 900000;
foreach ($results as $zip) {
$averageHouseValue['total'] += $zip['_source']['avgHomeValueIndex'];
$averageHouseValue['number']++;
$averageTurnover['total'] += $zip['_source']['turnoverWithinLastYear'];
$averageTurnover['number']++;
if ($zip['_source']['avgHomeValueIndex'] > $maxHouseValue) {
$maxHouseValue = $zip['_source']['averageHouseValue'];
}
if ($zip['_source']['averageHouseValue'] < $minHouseValue) {
$minHouseValue = $zip['_source']['averageHouseValue'];
}
}
$averageHouse = $averageHouseValue['total'] / $averageHouseValue['number'];
$averageTurn = $averageTurnover['total'] / $averageTurnover['number'];
$slidervalue = $req->get('slidervalue');
foreach ($results as $zip) {
$sliderInfo = $this->calculate($slidervalue);
$weight = $this->determineWeight($sliderInfo, $zip['_source']['avgHomeValueIndex'], $averageHouse, $maxHouseValue, $minHouseValue);
$responseObject[] = array('lat' => $zip['_source']['location']['lat'], 'lon' => $zip['_source']['location']['lon'], 'weight' => $weight);
}
return new Response(json_encode($responseObject), 200);
}
public function create()
{
$elasticsearch = new Client();
$sight = $this->sightRepository->getById(Input::get('sight_id'));
$time = Carbon::now($sight->city->timezone)->hour;
$searchQuery['index'] = 'sightseeing';
$searchQuery['type'] = 'sight';
$searchQuery['body'] = ['min_score' => 0.0001, 'query' => ['function_score' => ['query' => ['bool' => ['must' => [0 => ['range' => ['cost' => ['lte' => Input::get('cost')]]]], 'should' => [0 => ['range' => ['closing_hours' => ['gte' => $time]]], 1 => ['range' => ['opening_hours' => ['lte' => $time]]]]]], 'functions' => [0 => ['gauss' => ['location' => ['origin' => $sight->latitude . ',' . $sight->longitude, 'offset' => '0.5km', 'scale' => '0.1km', 'decay' => 0.33]]]]]]];
$suggestions = $elasticsearch->search($searchQuery);
$filteredSuggestions = array();
$filteredSuggestions['data'] = [];
foreach ($suggestions['hits']['hits'] as $suggestion) {
$suggestionAdded = false;
foreach ($suggestion['_source']['categories'] as $category) {
if (in_array($category['id'], Input::get('categories')) && !$suggestionAdded && $suggestion['_id'] != Input::get('sight_id')) {
$filteredSuggestions['data'][] = ['id' => $suggestion['_id'], 'score' => $suggestion['_score']];
$suggestionAdded = true;
}
}
}
$maxScore = 0;
foreach ($filteredSuggestions['data'] as $suggestion) {
if ($suggestion['score'] > $maxScore) {
$maxScore = $suggestion['score'];
}
}
$filteredSuggestions['max_score'] = $maxScore;
return $filteredSuggestions;
}
public function findProducts(ListingFilter $filter)
{
$must = [];
if ($filter->getEshop() !== null) {
$must[] = ["term" => [ProductMeta::ESHOP_ID => (string) $filter->getEshop()->getId()]];
}
if ($filter->getCategory() !== null) {
/** @var Category[] $childrenCategories */
$childrenCategories = $this->categoryRepository->find([CategoryMeta::PATH => $filter->getCategory()->getId()]);
$must[] = ["terms" => [ProductMeta::CATEGORY_IDS => array_merge([(string) $filter->getCategory()->getId()], array_map(function (Category $category) {
return (string) $category->getId();
}, $childrenCategories))]];
}
$body = ["query" => ["filtered" => ["filter" => ["bool" => ["must" => $must]]]], "from" => $filter->getOffset(), "size" => $filter->getLimit(), "sort" => ["_score" => "desc"]];
if ($filter->getQ() !== null) {
$body["query"]["filtered"]["query"] = ["multi_match" => ["query" => $filter->getQ(), "fields" => [ProductMeta::NAME . "^5", ProductMeta::LONG_NAME . "^5", ProductMeta::DESCRIPTION, ProductMeta::MANUFACTURER . "^2", ProductMeta::BRAND . "^2", ProductMeta::ESHOP . "." . EshopMeta::NAME . "^2"]]];
}
// if (empty($body["query"]["filtered"]["filter"]["bool"]["must"])) {
// unset($body["query"]["filtered"]["filter"]);
// }
$response = $this->elasticsearch->search(["index" => $this->catalogIndexAliasName, "type" => ProductMeta::SHORT_NAME, "body" => $body]);
if (!isset($response["hits"]["hits"])) {
throw new \RuntimeException("Response does not have hits->hits. Got: " . json_encode($response));
}
$products = [];
foreach ($response["hits"]["hits"] as $hit) {
$products[] = ProductMeta::fromArray($hit["_source"], "json:");
}
return $products;
}
/**
* Handle index creation command
*
* @param Client $client
* @param string $index
*/
public function handle(Client $client, $index)
{
$config = $this->configurationRepository->get($index);
if (null === $config) {
throw new \InvalidArgumentException();
}
$client->indices()->create(['index' => $index, 'body' => $config]);
}
/**
* 查询 结果
*
* @param string $index 索引
* @param string $type 类型
* @param string $body 查询字符串
* @param array $attrs 额外查询参数
*
* @return mixed
*/
public function search($index, $type, $body, $attrs = [])
{
$query = ['index' => $index, 'type' => $type, 'body' => $body];
if (!empty($attrs)) {
$query = array_merge($attrs, $query);
}
return $this->esClient->search($query);
}
/**
* Handle index creation command
*
* @param Client $client
* @param string $index
*/
public function handle($client, $index)
{
$config = $this->configurationRepository->get($index);
if (null === $client || null === $config) {
throw new \InvalidArgumentException();
}
$client->indices()->putSettings(['index' => $index, 'body' => ['settings' => $this->extractSettings($config)]]);
}
/**
* Handle index creation command
*
* @param Client $client
* @param string $index
*/
public function handle(Client $client, $index, $type)
{
$config = $this->configurationRepository->get($index);
if ($this->isInvalid($config, $type)) {
throw new \InvalidArgumentException();
}
$client->indices()->putMapping(['index' => $index, 'type' => $type, 'body' => [$type => $config['mappings'][$type]]]);
}
/**
* Index all products
*
* @return number
*/
public function indexAll()
{
$products = $this->database->table('product')->fetchAll();
foreach ($products as $product) {
$this->es->index(['index' => $this->indexName, 'type' => 'product', 'id' => $product['id'], 'body' => $product->toArray()]);
}
return count($products);
}
/**
* @covers Fabfuel\Prophiler\Decorator\Elasticsearch\ClientDecorator::__call
*/
public function testCall()
{
$payload = ['lorem' => 'ipsum'];
$benchmark = $this->getMock('Fabfuel\\Prophiler\\Benchmark\\BenchmarkInterface');
$this->client->expects($this->once())->method('get')->with($payload);
$this->profiler->expects($this->once())->method('start')->with('ElasticMock::get', [$payload], 'Elasticsearch')->willReturn($benchmark);
$this->profiler->expects($this->once())->method('stop')->with($benchmark);
$this->decorator->get($payload);
}
/**
*
* {@inheritdoc}
*
*/
public function getParameters(View $view, FormFactoryInterface $formFactoty, Request $request)
{
$searchQuery = ['index' => $view->getContentType()->getEnvironment()->getAlias(), 'type' => $view->getContentType()->getName(), 'search_type' => 'count', 'body' => $view->getOptions()['aggsQuery']];
$retDoc = $this->client->search($searchQuery);
foreach (explode('.', $view->getOptions()['pathToBuckets']) as $attribute) {
$retDoc = $retDoc[$attribute];
}
return ['keywords' => $retDoc, 'view' => $view, 'contentType' => $view->getContentType(), 'environment' => $view->getContentType()->getEnvironment()];
}
public static function addDocuments(\ElasticSearch\Client $client, $num = 3, $tag = 'cool')
{
$options = array('refresh' => true);
while ($num-- > 0) {
$doc = array('title' => "One cool document {$tag}", 'rank' => rand(1, 10));
$client->index($doc, $num + 1, $options);
}
return $client;
}
/**
* Create the synonyms index for a store id.
*
* @param integer $storeId Store id.
* @param string[] $synonyms Raw synonyms list.
* @param string[] $expansions Raw expansions list.
*
* @return void
*/
public function reindex($storeId, $synonyms, $expansions)
{
$indexIdentifier = ThesaurusIndex::INDEX_IDENTIER;
$indexName = $this->indexSettingsHelper->createIndexNameFromIdentifier($indexIdentifier, $storeId);
$indexAlias = $this->indexSettingsHelper->getIndexAliasFromIdentifier($indexIdentifier, $storeId);
$indexSettings = ['settings' => $this->getIndexSettings($synonyms, $expansions)];
$this->client->indices()->create(['index' => $indexName, 'body' => $indexSettings]);
$this->indexManager->proceedIndexInstall($indexName, $indexAlias);
$this->cacheHelper->cleanIndexCache(ThesaurusIndex::INDEX_IDENTIER, $storeId);
}
/**
* Execute the request by elasticsearch client
*
* @param Client $client
* @return ResponseInterface
*/
public function executeByElasticClient(Client $client)
{
$responseClass = $this->getResponseClassOfRequest();
/** @var IndexResponseInterface $response */
$response = new $responseClass();
$rawResult = RawResponse::build($client->index($this->toElasticClient()));
$response = $response->build($rawResult);
$this->getDocument()->setId($response->id());
return $response;
}
public function testCreateJob()
{
$job = self::$jobFactory->create(uniqid());
$id = self::$jobMapper->create($job);
$res = self::$client->get(['index' => self::$index->getIndexNameCurrent(), 'type' => 'jobs', 'id' => $id]);
$resJob = $res['_source'];
$job = self::$jobMapper->get($id);
$this->assertJob($job, $resJob);
$this->assertEquals($job->getVersion(), $res['_version']);
}
/**
* @Route("/")
* @Template("DashboardMainBundle:Default:index.html.twig")
*/
public function index()
{
$params = array();
$params['hosts'] = array('127.0.0.1:9200');
$client = new Elasticsearch\Client($params);
$params = array("index" => "dash-mail-*", "type" => "mail", "body" => array("query" => array("filtered" => array("filter" => array("bool" => array("must" => array(array("missing" => array("field" => "flags")), array("term" => array("folderFullName" => "INBOX")))))))));
$results_mail = $client->search($params);
$params = array("index" => "dash-rss-*", "type" => "page");
$results_rss = $client->count($params);
$params = array("index" => "dash-twitter-*", "type" => "status");
$results_twitter = $client->count($params);
return array("mail_unread" => $results_mail['hits']['total'], "rss_total" => $results_rss['count'], "twitter_total" => $results_twitter['count']);
}
/**
* Execute the request by elasticsearch client
*
* @param Client $client
* @return ResponseInterface
*/
public function executeByElasticClient(Client $client)
{
$params = $this->toElasticClient();
$responseClass = $this->getResponseClassOfRequest();
/** @var GetResponseInterface $response */
$response = new $responseClass();
$result = RawResponse::build($client->get($params));
if (null !== $this->document) {
$response->setDocument($this->document);
}
$response->build($result);
return $response;
}
private function searchBlog(Criteria $criteria, Struct\ProductContextInterface $context)
{
/**@var $condition SearchTermCondition*/
$condition = $criteria->getCondition('search');
$query = $this->createMultiMatchQuery($condition);
$search = new Search();
$search->addQuery($query);
$search->setFrom(0)->setSize(5);
$index = $this->indexFactory->createShopIndex($context->getShop());
$params = ['index' => $index->getName(), 'type' => 'blog', 'body' => $search->toArray()];
$raw = $this->client->search($params);
return $this->createBlogStructs($raw);
}
/**
* @param $glassNamePartial
* @param int $limit
* @return array []
*/
public function glass($glassNamePartial, $limit = 10)
{
$params = ['index' => ElasticSearch::INDEX, 'type' => 'supply', 'body' => ['query' => ['match_phrase_prefix' => ['polishName' => ['query' => $glassNamePartial, 'max_expansions' => 10]]], 'filter' => ['prefix' => ['_id' => 'glass.']]]];
$results = $this->client->search($params);
if ($results['hits']['total'] === 0) {
return [];
}
$liquids = [];
foreach ($results['hits']['hits'] as $result) {
$liquids[] = new Supply($result['_id'], $result['_source']['polishName']);
}
return $liquids;
}
/**
* {@inheritdoc}
*/
public function getStatus()
{
$data = $this->client->info();
$version = $data['version'];
unset($data['version']);
foreach ($version as $prop => $value) {
$data['version:' . $prop] = $value;
}
$ret = [];
foreach ($data as $key => $value) {
$ret[] = [$key, $value];
}
return $ret;
}
/**
*
* {@inheritdoc}
*
*/
public function getParameters(View $view, FormFactoryInterface $formFactoty, Request $request)
{
try {
$renderQuery = $this->twig->createTemplate($view->getOptions()['body'])->render(['view' => $view, 'contentType' => $view->getContentType(), 'environment' => $view->getContentType()->getEnvironment()]);
} catch (\Exception $e) {
$renderQuery = "{}";
}
$searchQuery = ['index' => $view->getContentType()->getEnvironment()->getAlias(), 'type' => $view->getContentType()->getName(), 'body' => $renderQuery, 'size' => $view->getOptions()['size']];
$result = $this->client->search($searchQuery);
try {
$render = $this->twig->createTemplate($view->getOptions()['template'])->render(['view' => $view, 'contentType' => $view->getContentType(), 'environment' => $view->getContentType()->getEnvironment(), 'result' => $result]);
} catch (\Exception $e) {
$render = "Something went wrong with the template of the view " . $view->getName() . " for the content type " . $view->getContentType()->getName() . " (" . $e->getMessage() . ")";
}
return ['render' => $render, 'view' => $view, 'contentType' => $view->getContentType(), 'environment' => $view->getContentType()->getEnvironment()];
}
/**
* @param Criteria $criteria
* @return SearchResultSlice
*/
public function search(Criteria $criteria)
{
$results = $this->client->search($this->parametersBuiler->createParameters($criteria));
if ($results['hits']['total'] === 0) {
return new SearchResultSlice($criteria, [], 0);
}
$recipes = [];
foreach ($results['hits']['hits'] as $result) {
$recipe = new SearchEngine\Result\Recipe($result['_id'], $result['_source']['name'], $result['_source']['description']['text']);
if (!is_null($result['_source']['publicationDate'])) {
$recipe->publishedAt(new \DateTimeImmutable($result['_source']['publicationDate']));
}
$recipes[] = $recipe;
}
return new SearchResultSlice($criteria, $recipes, $results['hits']['total']);
}
/**
* {@inheritdoc}
*/
public function hydrate(array $elasticResult, ProductNumberSearchResult $result, Criteria $criteria, ShopContextInterface $context)
{
if (!isset($elasticResult['aggregations'])) {
return;
}
if (!isset($elasticResult['aggregations']['agg_properties'])) {
return;
}
$data = $elasticResult['aggregations']['agg_properties']['buckets'];
$ids = array_column($data, 'key');
if (empty($ids)) {
return;
}
$groupIds = $this->getGroupIds($ids);
$search = new Search();
$search->addFilter(new IdsFilter($groupIds));
$search->addFilter(new TermFilter('filterable', 1));
$search->addSort(new FieldSort('name'));
$index = $this->indexFactory->createShopIndex($context->getShop());
$data = $this->client->search(['index' => $index->getName(), 'type' => PropertyMapping::TYPE, 'body' => $search->toArray()]);
$data = $data['hits']['hits'];
$properties = $this->hydrateProperties($data, $ids);
$actives = $this->getFilteredValues($criteria);
$criteriaPart = $this->createCollectionResult($properties, $actives);
$result->addFacet($criteriaPart);
}
public function testMapFields()
{
$client = \ElasticSearch\Client::connection(array('index' => 'test-index', 'type' => 'test-type'));
$client->index(array('tweet' => 'ElasticSearch is awesome'));
$response = $client->map(array('tweet' => array('type' => 'string')));
$this->assert->array($response)->isNotEmpty();
}
/**
* @param Orchestration $orchestration
* @param int $offset
* @param int $limit
* @return Metadata\Job[]
*/
private function loadOrchestrationSuccessJobs(Orchestration $orchestration, $offset = 0, $limit = 25)
{
/**
* @var ComponentIndex $config
*/
$config = $this->getContainer()->get('syrup.elasticsearch.current_component_index');
$orchestrations = $this->loadActiveOrchestrationIds();
$filter = [];
$filter[] = ['term' => ['status' => Metadata\Job::STATUS_SUCCESS]];
$filter[] = ['terms' => ['params.orchestration.id' => array($orchestration->getId())]];
$query = ['match_all' => []];
$params = [];
$params['index'] = $config->getIndexPrefix() . '_syrup_' . KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME;
$params['body'] = ['from' => (int) $offset, 'size' => (int) $limit, 'query' => ['filtered' => ['filter' => ['bool' => ['must' => $filter]], 'query' => $query]], 'sort' => ['id' => ['order' => 'desc']]];
$results = [];
$hits = $this->elasticSearch->search($params);
foreach ($hits['hits']['hits'] as $hit) {
$res = $hit['_source'];
$res['_index'] = $hit['_index'];
$res['_type'] = $hit['_type'];
$res['id'] = (int) $res['id'];
$results[] = $res;
}
$objectEncryptor = $this->objectEncryptor;
return array_map(function ($line) use($objectEncryptor) {
return new Metadata\Job($objectEncryptor, $line, $line['_index'], $line['_type']);
}, $results);
}
/**
* {@inheritdoc}
*/
public function query($string, $offset, $perPage, SearchEngineOptions $options = null)
{
$options = $options ?: new SearchEngineOptions();
$context = $this->context_factory->createContext($options);
/** @var QueryCompiler $query_compiler */
$query_compiler = $this->app['query_compiler'];
$recordQuery = $query_compiler->compile($string, $context);
$params = $this->createRecordQueryParams($recordQuery, $options, null);
$params['body']['from'] = $offset;
$params['body']['size'] = $perPage;
if ($this->options->getHighlight()) {
$params['body']['highlight'] = $this->buildHighlightRules($context);
}
if ($aggs = $this->getAggregationQueryParams($options)) {
$params['body']['aggs'] = $aggs;
}
$res = $this->client->search($params);
$results = new ArrayCollection();
$n = 0;
foreach ($res['hits']['hits'] as $hit) {
$results[] = ElasticsearchRecordHydrator::hydrate($hit, $n++);
}
/** @var FacetsResponse $facets */
$facets = $this->facetsResponseFactory->__invoke($res);
$query['ast'] = $query_compiler->parse($string)->dump();
$query['query_main'] = $recordQuery;
$query['query'] = $params['body'];
$query['query_string'] = json_encode($params['body']);
return new SearchEngineResult($results, json_encode($query), $res['took'], $offset, $res['hits']['total'], $res['hits']['total'], null, null, $facets->getAsSuggestions(), [], $this->indexName, $facets);
}
public function createIndex($settings = null, $mappings = null)
{
// Assemble new index name
$nextIndexNumber = 1;
$lastIndexName = $this->getLastIndexName();
if (null != $lastIndexName) {
$lastIndexNameArr = explode('_', $lastIndexName);
$nextIndexNumber = array_pop($lastIndexNameArr) + 1;
}
$nextIndexName = $this->getIndexName() . '_' . date('Y') . '_' . $nextIndexNumber;
// Create new index
$params['index'] = $nextIndexName;
if (null != $settings) {
$params['body']['settings'] = $settings;
}
if (null != $mappings) {
$params['body']['mappings'] = $mappings;
}
$this->client->indices()->create($params);
// Update aliases
$params = [];
$params['body'] = ['actions' => [['add' => ['index' => $nextIndexName, 'alias' => $this->getIndexNameCurrent()]], ['add' => ['index' => $nextIndexName, 'alias' => $this->getIndexName()]]]];
if (null != $lastIndexName) {
array_unshift($params['body']['actions'], ['remove' => ['index' => $lastIndexName, 'alias' => $this->getIndexNameCurrent()]]);
}
$this->client->indices()->updateAliases($params);
return $nextIndexName;
}
/**
* @param string $snsid
* @param int $status
*
* @return array
*/
protected function updateUserStatus($snsid, $status)
{
assert(is_numeric($status));
$params = ['index' => $this->index, 'type' => $this->type, 'id' => $snsid, 'body' => '{"doc": {"status": "' . $status . '"}}'];
try {
$ret = $this->client->update($params);
return ['snsid' => $snsid, 'version' => $ret['_version']];
} catch (\Exception $e) {
$errMsg = $e->getMessage();
$decodedArray = json_decode($errMsg, true);
if (is_array($decodedArray)) {
return ['snsid' => $snsid, 'error' => $decodedArray['status']];
}
return ['snsid' => $snsid, 'error' => $errMsg];
}
}
public function get($jobId)
{
$indices = $this->index->getIndices();
$docs = [];
foreach ($indices as $index) {
$docs[] = ['_index' => $index, '_type' => 'jobs', '_id' => $jobId];
}
$i = 0;
$prevException = null;
while ($i < 5) {
try {
$result = $this->client->mget(['body' => ['docs' => $docs]]);
foreach ($result['docs'] as $doc) {
if ($doc['found']) {
return new Job($this->configEncryptor, $doc['_source'], $doc['_index'], $doc['_type'], $doc['_version']);
}
}
} catch (ServerErrorResponseException $e) {
// ES server error, try again
$this->log('error', 'Elastic server error response', ['attemptNo' => $i, 'jobId' => $jobId, 'exception' => $e]);
$prevException = $e;
}
sleep(1 + intval(pow(2, $i) / 2));
$i++;
}
$this->log('alert', sprintf("Error getting job id '%s'", $jobId), ['exception' => $prevException]);
return null;
}
/**
* @iterations 1000
* @group large
*/
public function asyncLarge()
{
$asyncDoc = $this->largeDocument;
$asyncDoc['client']['future'] = 'lazy';
$response = $this->client->index($asyncDoc);
$response = $response['body']['created'];
}
