/** * Flushes the items in this Bulk object to Elasticsearch. If there are no items, this method does nothing. * * (In particular, note that if no items are in the Bulk object, the bulk flush event is not fired.) * * @throws PartialFailureException */ public function flush() { // We might have nothing to do if ($this->getItemCount() == 0) { return; } $this->dispatchEvent(Events::BULK_FLUSH, new BulkFlushEvent($this->index, $this->type, $this->getItemCount())); $params = ['index' => $this->index, 'type' => $this->type, 'body' => $this->itemList]; $result = $this->elasticSearch->bulk($params); if (count($result['items']) != $this->itemCount) { $expected = $this->itemCount; $actual = count($result['items']); throw new PartialFailureException("The wrong number of items was stored; expected {$expected}, but stored {$actual}"); } if (boolval($result['errors'])) { $errorList = []; foreach ($result['items'] as $item) { if ($item['index']['status'] < 200 || $item['index']['status'] >= 300) { $errorList[] = $item; } } throw new PartialFailureException("Some items failed. " . json_encode($errorList)); } $this->clear(); }
public function main(Request $req, Application $app) { $client = new Client(); $params = array('index' => 'hrqls', 'type' => 'houseData', 'body' => ['from' => 0, 'size' => 100, 'filter' => array('range' => array('avgHomeValueIndex' => array('gte' => 0))), 'query' => ['match' => ['state' => 'Virginia']]]); $results = $client->search($params)['hits']['hits']; $responseObject = []; $averageHouseValue = array('total' => 0, 'number' => 0); $averageTurnover = array('total' => 0, 'number' => 0); $maxHouseValue = 0; $minHouseValue = 900000; foreach ($results as $zip) { $averageHouseValue['total'] += $zip['_source']['avgHomeValueIndex']; $averageHouseValue['number']++; $averageTurnover['total'] += $zip['_source']['turnoverWithinLastYear']; $averageTurnover['number']++; if ($zip['_source']['avgHomeValueIndex'] > $maxHouseValue) { $maxHouseValue = $zip['_source']['averageHouseValue']; } if ($zip['_source']['averageHouseValue'] < $minHouseValue) { $minHouseValue = $zip['_source']['averageHouseValue']; } } $averageHouse = $averageHouseValue['total'] / $averageHouseValue['number']; $averageTurn = $averageTurnover['total'] / $averageTurnover['number']; $slidervalue = $req->get('slidervalue'); foreach ($results as $zip) { $sliderInfo = $this->calculate($slidervalue); $weight = $this->determineWeight($sliderInfo, $zip['_source']['avgHomeValueIndex'], $averageHouse, $maxHouseValue, $minHouseValue); $responseObject[] = array('lat' => $zip['_source']['location']['lat'], 'lon' => $zip['_source']['location']['lon'], 'weight' => $weight); } return new Response(json_encode($responseObject), 200); }
public function create() { $elasticsearch = new Client(); $sight = $this->sightRepository->getById(Input::get('sight_id')); $time = Carbon::now($sight->city->timezone)->hour; $searchQuery['index'] = 'sightseeing'; $searchQuery['type'] = 'sight'; $searchQuery['body'] = ['min_score' => 0.0001, 'query' => ['function_score' => ['query' => ['bool' => ['must' => [0 => ['range' => ['cost' => ['lte' => Input::get('cost')]]]], 'should' => [0 => ['range' => ['closing_hours' => ['gte' => $time]]], 1 => ['range' => ['opening_hours' => ['lte' => $time]]]]]], 'functions' => [0 => ['gauss' => ['location' => ['origin' => $sight->latitude . ',' . $sight->longitude, 'offset' => '0.5km', 'scale' => '0.1km', 'decay' => 0.33]]]]]]]; $suggestions = $elasticsearch->search($searchQuery); $filteredSuggestions = array(); $filteredSuggestions['data'] = []; foreach ($suggestions['hits']['hits'] as $suggestion) { $suggestionAdded = false; foreach ($suggestion['_source']['categories'] as $category) { if (in_array($category['id'], Input::get('categories')) && !$suggestionAdded && $suggestion['_id'] != Input::get('sight_id')) { $filteredSuggestions['data'][] = ['id' => $suggestion['_id'], 'score' => $suggestion['_score']]; $suggestionAdded = true; } } } $maxScore = 0; foreach ($filteredSuggestions['data'] as $suggestion) { if ($suggestion['score'] > $maxScore) { $maxScore = $suggestion['score']; } } $filteredSuggestions['max_score'] = $maxScore; return $filteredSuggestions; }
public function findProducts(ListingFilter $filter) { $must = []; if ($filter->getEshop() !== null) { $must[] = ["term" => [ProductMeta::ESHOP_ID => (string) $filter->getEshop()->getId()]]; } if ($filter->getCategory() !== null) { /** @var Category[] $childrenCategories */ $childrenCategories = $this->categoryRepository->find([CategoryMeta::PATH => $filter->getCategory()->getId()]); $must[] = ["terms" => [ProductMeta::CATEGORY_IDS => array_merge([(string) $filter->getCategory()->getId()], array_map(function (Category $category) { return (string) $category->getId(); }, $childrenCategories))]]; } $body = ["query" => ["filtered" => ["filter" => ["bool" => ["must" => $must]]]], "from" => $filter->getOffset(), "size" => $filter->getLimit(), "sort" => ["_score" => "desc"]]; if ($filter->getQ() !== null) { $body["query"]["filtered"]["query"] = ["multi_match" => ["query" => $filter->getQ(), "fields" => [ProductMeta::NAME . "^5", ProductMeta::LONG_NAME . "^5", ProductMeta::DESCRIPTION, ProductMeta::MANUFACTURER . "^2", ProductMeta::BRAND . "^2", ProductMeta::ESHOP . "." . EshopMeta::NAME . "^2"]]]; } // if (empty($body["query"]["filtered"]["filter"]["bool"]["must"])) { // unset($body["query"]["filtered"]["filter"]); // } $response = $this->elasticsearch->search(["index" => $this->catalogIndexAliasName, "type" => ProductMeta::SHORT_NAME, "body" => $body]); if (!isset($response["hits"]["hits"])) { throw new \RuntimeException("Response does not have hits->hits. Got: " . json_encode($response)); } $products = []; foreach ($response["hits"]["hits"] as $hit) { $products[] = ProductMeta::fromArray($hit["_source"], "json:"); } return $products; }
/** * Handle index creation command * * @param Client $client * @param string $index */ public function handle(Client $client, $index) { $config = $this->configurationRepository->get($index); if (null === $config) { throw new \InvalidArgumentException(); } $client->indices()->create(['index' => $index, 'body' => $config]); }
/** * 查询 结果 * * @param string $index 索引 * @param string $type 类型 * @param string $body 查询字符串 * @param array $attrs 额外查询参数 * * @return mixed */ public function search($index, $type, $body, $attrs = []) { $query = ['index' => $index, 'type' => $type, 'body' => $body]; if (!empty($attrs)) { $query = array_merge($attrs, $query); } return $this->esClient->search($query); }
/** * Handle index creation command * * @param Client $client * @param string $index */ public function handle($client, $index) { $config = $this->configurationRepository->get($index); if (null === $client || null === $config) { throw new \InvalidArgumentException(); } $client->indices()->putSettings(['index' => $index, 'body' => ['settings' => $this->extractSettings($config)]]); }
/** * Handle index creation command * * @param Client $client * @param string $index */ public function handle(Client $client, $index, $type) { $config = $this->configurationRepository->get($index); if ($this->isInvalid($config, $type)) { throw new \InvalidArgumentException(); } $client->indices()->putMapping(['index' => $index, 'type' => $type, 'body' => [$type => $config['mappings'][$type]]]); }
/** * Index all products * * @return number */ public function indexAll() { $products = $this->database->table('product')->fetchAll(); foreach ($products as $product) { $this->es->index(['index' => $this->indexName, 'type' => 'product', 'id' => $product['id'], 'body' => $product->toArray()]); } return count($products); }
/** * @covers Fabfuel\Prophiler\Decorator\Elasticsearch\ClientDecorator::__call */ public function testCall() { $payload = ['lorem' => 'ipsum']; $benchmark = $this->getMock('Fabfuel\\Prophiler\\Benchmark\\BenchmarkInterface'); $this->client->expects($this->once())->method('get')->with($payload); $this->profiler->expects($this->once())->method('start')->with('ElasticMock::get', [$payload], 'Elasticsearch')->willReturn($benchmark); $this->profiler->expects($this->once())->method('stop')->with($benchmark); $this->decorator->get($payload); }
/** * * {@inheritdoc} * */ public function getParameters(View $view, FormFactoryInterface $formFactoty, Request $request) { $searchQuery = ['index' => $view->getContentType()->getEnvironment()->getAlias(), 'type' => $view->getContentType()->getName(), 'search_type' => 'count', 'body' => $view->getOptions()['aggsQuery']]; $retDoc = $this->client->search($searchQuery); foreach (explode('.', $view->getOptions()['pathToBuckets']) as $attribute) { $retDoc = $retDoc[$attribute]; } return ['keywords' => $retDoc, 'view' => $view, 'contentType' => $view->getContentType(), 'environment' => $view->getContentType()->getEnvironment()]; }
public static function addDocuments(\ElasticSearch\Client $client, $num = 3, $tag = 'cool') { $options = array('refresh' => true); while ($num-- > 0) { $doc = array('title' => "One cool document {$tag}", 'rank' => rand(1, 10)); $client->index($doc, $num + 1, $options); } return $client; }
/** * Create the synonyms index for a store id. * * @param integer $storeId Store id. * @param string[] $synonyms Raw synonyms list. * @param string[] $expansions Raw expansions list. * * @return void */ public function reindex($storeId, $synonyms, $expansions) { $indexIdentifier = ThesaurusIndex::INDEX_IDENTIER; $indexName = $this->indexSettingsHelper->createIndexNameFromIdentifier($indexIdentifier, $storeId); $indexAlias = $this->indexSettingsHelper->getIndexAliasFromIdentifier($indexIdentifier, $storeId); $indexSettings = ['settings' => $this->getIndexSettings($synonyms, $expansions)]; $this->client->indices()->create(['index' => $indexName, 'body' => $indexSettings]); $this->indexManager->proceedIndexInstall($indexName, $indexAlias); $this->cacheHelper->cleanIndexCache(ThesaurusIndex::INDEX_IDENTIER, $storeId); }
/** * Execute the request by elasticsearch client * * @param Client $client * @return ResponseInterface */ public function executeByElasticClient(Client $client) { $responseClass = $this->getResponseClassOfRequest(); /** @var IndexResponseInterface $response */ $response = new $responseClass(); $rawResult = RawResponse::build($client->index($this->toElasticClient())); $response = $response->build($rawResult); $this->getDocument()->setId($response->id()); return $response; }
public function testCreateJob() { $job = self::$jobFactory->create(uniqid()); $id = self::$jobMapper->create($job); $res = self::$client->get(['index' => self::$index->getIndexNameCurrent(), 'type' => 'jobs', 'id' => $id]); $resJob = $res['_source']; $job = self::$jobMapper->get($id); $this->assertJob($job, $resJob); $this->assertEquals($job->getVersion(), $res['_version']); }
/** * @Route("/") * @Template("DashboardMainBundle:Default:index.html.twig") */ public function index() { $params = array(); $params['hosts'] = array('127.0.0.1:9200'); $client = new Elasticsearch\Client($params); $params = array("index" => "dash-mail-*", "type" => "mail", "body" => array("query" => array("filtered" => array("filter" => array("bool" => array("must" => array(array("missing" => array("field" => "flags")), array("term" => array("folderFullName" => "INBOX"))))))))); $results_mail = $client->search($params); $params = array("index" => "dash-rss-*", "type" => "page"); $results_rss = $client->count($params); $params = array("index" => "dash-twitter-*", "type" => "status"); $results_twitter = $client->count($params); return array("mail_unread" => $results_mail['hits']['total'], "rss_total" => $results_rss['count'], "twitter_total" => $results_twitter['count']); }
/** * Execute the request by elasticsearch client * * @param Client $client * @return ResponseInterface */ public function executeByElasticClient(Client $client) { $params = $this->toElasticClient(); $responseClass = $this->getResponseClassOfRequest(); /** @var GetResponseInterface $response */ $response = new $responseClass(); $result = RawResponse::build($client->get($params)); if (null !== $this->document) { $response->setDocument($this->document); } $response->build($result); return $response; }
private function searchBlog(Criteria $criteria, Struct\ProductContextInterface $context) { /**@var $condition SearchTermCondition*/ $condition = $criteria->getCondition('search'); $query = $this->createMultiMatchQuery($condition); $search = new Search(); $search->addQuery($query); $search->setFrom(0)->setSize(5); $index = $this->indexFactory->createShopIndex($context->getShop()); $params = ['index' => $index->getName(), 'type' => 'blog', 'body' => $search->toArray()]; $raw = $this->client->search($params); return $this->createBlogStructs($raw); }
/** * @param $glassNamePartial * @param int $limit * @return array [] */ public function glass($glassNamePartial, $limit = 10) { $params = ['index' => ElasticSearch::INDEX, 'type' => 'supply', 'body' => ['query' => ['match_phrase_prefix' => ['polishName' => ['query' => $glassNamePartial, 'max_expansions' => 10]]], 'filter' => ['prefix' => ['_id' => 'glass.']]]]; $results = $this->client->search($params); if ($results['hits']['total'] === 0) { return []; } $liquids = []; foreach ($results['hits']['hits'] as $result) { $liquids[] = new Supply($result['_id'], $result['_source']['polishName']); } return $liquids; }
/** * {@inheritdoc} */ public function getStatus() { $data = $this->client->info(); $version = $data['version']; unset($data['version']); foreach ($version as $prop => $value) { $data['version:' . $prop] = $value; } $ret = []; foreach ($data as $key => $value) { $ret[] = [$key, $value]; } return $ret; }
/** * * {@inheritdoc} * */ public function getParameters(View $view, FormFactoryInterface $formFactoty, Request $request) { try { $renderQuery = $this->twig->createTemplate($view->getOptions()['body'])->render(['view' => $view, 'contentType' => $view->getContentType(), 'environment' => $view->getContentType()->getEnvironment()]); } catch (\Exception $e) { $renderQuery = "{}"; } $searchQuery = ['index' => $view->getContentType()->getEnvironment()->getAlias(), 'type' => $view->getContentType()->getName(), 'body' => $renderQuery, 'size' => $view->getOptions()['size']]; $result = $this->client->search($searchQuery); try { $render = $this->twig->createTemplate($view->getOptions()['template'])->render(['view' => $view, 'contentType' => $view->getContentType(), 'environment' => $view->getContentType()->getEnvironment(), 'result' => $result]); } catch (\Exception $e) { $render = "Something went wrong with the template of the view " . $view->getName() . " for the content type " . $view->getContentType()->getName() . " (" . $e->getMessage() . ")"; } return ['render' => $render, 'view' => $view, 'contentType' => $view->getContentType(), 'environment' => $view->getContentType()->getEnvironment()]; }
/** * @param Criteria $criteria * @return SearchResultSlice */ public function search(Criteria $criteria) { $results = $this->client->search($this->parametersBuiler->createParameters($criteria)); if ($results['hits']['total'] === 0) { return new SearchResultSlice($criteria, [], 0); } $recipes = []; foreach ($results['hits']['hits'] as $result) { $recipe = new SearchEngine\Result\Recipe($result['_id'], $result['_source']['name'], $result['_source']['description']['text']); if (!is_null($result['_source']['publicationDate'])) { $recipe->publishedAt(new \DateTimeImmutable($result['_source']['publicationDate'])); } $recipes[] = $recipe; } return new SearchResultSlice($criteria, $recipes, $results['hits']['total']); }
/** * {@inheritdoc} */ public function hydrate(array $elasticResult, ProductNumberSearchResult $result, Criteria $criteria, ShopContextInterface $context) { if (!isset($elasticResult['aggregations'])) { return; } if (!isset($elasticResult['aggregations']['agg_properties'])) { return; } $data = $elasticResult['aggregations']['agg_properties']['buckets']; $ids = array_column($data, 'key'); if (empty($ids)) { return; } $groupIds = $this->getGroupIds($ids); $search = new Search(); $search->addFilter(new IdsFilter($groupIds)); $search->addFilter(new TermFilter('filterable', 1)); $search->addSort(new FieldSort('name')); $index = $this->indexFactory->createShopIndex($context->getShop()); $data = $this->client->search(['index' => $index->getName(), 'type' => PropertyMapping::TYPE, 'body' => $search->toArray()]); $data = $data['hits']['hits']; $properties = $this->hydrateProperties($data, $ids); $actives = $this->getFilteredValues($criteria); $criteriaPart = $this->createCollectionResult($properties, $actives); $result->addFacet($criteriaPart); }
public function testMapFields() { $client = \ElasticSearch\Client::connection(array('index' => 'test-index', 'type' => 'test-type')); $client->index(array('tweet' => 'ElasticSearch is awesome')); $response = $client->map(array('tweet' => array('type' => 'string'))); $this->assert->array($response)->isNotEmpty(); }
/** * @param Orchestration $orchestration * @param int $offset * @param int $limit * @return Metadata\Job[] */ private function loadOrchestrationSuccessJobs(Orchestration $orchestration, $offset = 0, $limit = 25) { /** * @var ComponentIndex $config */ $config = $this->getContainer()->get('syrup.elasticsearch.current_component_index'); $orchestrations = $this->loadActiveOrchestrationIds(); $filter = []; $filter[] = ['term' => ['status' => Metadata\Job::STATUS_SUCCESS]]; $filter[] = ['terms' => ['params.orchestration.id' => array($orchestration->getId())]]; $query = ['match_all' => []]; $params = []; $params['index'] = $config->getIndexPrefix() . '_syrup_' . KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME; $params['body'] = ['from' => (int) $offset, 'size' => (int) $limit, 'query' => ['filtered' => ['filter' => ['bool' => ['must' => $filter]], 'query' => $query]], 'sort' => ['id' => ['order' => 'desc']]]; $results = []; $hits = $this->elasticSearch->search($params); foreach ($hits['hits']['hits'] as $hit) { $res = $hit['_source']; $res['_index'] = $hit['_index']; $res['_type'] = $hit['_type']; $res['id'] = (int) $res['id']; $results[] = $res; } $objectEncryptor = $this->objectEncryptor; return array_map(function ($line) use($objectEncryptor) { return new Metadata\Job($objectEncryptor, $line, $line['_index'], $line['_type']); }, $results); }
/** * {@inheritdoc} */ public function query($string, $offset, $perPage, SearchEngineOptions $options = null) { $options = $options ?: new SearchEngineOptions(); $context = $this->context_factory->createContext($options); /** @var QueryCompiler $query_compiler */ $query_compiler = $this->app['query_compiler']; $recordQuery = $query_compiler->compile($string, $context); $params = $this->createRecordQueryParams($recordQuery, $options, null); $params['body']['from'] = $offset; $params['body']['size'] = $perPage; if ($this->options->getHighlight()) { $params['body']['highlight'] = $this->buildHighlightRules($context); } if ($aggs = $this->getAggregationQueryParams($options)) { $params['body']['aggs'] = $aggs; } $res = $this->client->search($params); $results = new ArrayCollection(); $n = 0; foreach ($res['hits']['hits'] as $hit) { $results[] = ElasticsearchRecordHydrator::hydrate($hit, $n++); } /** @var FacetsResponse $facets */ $facets = $this->facetsResponseFactory->__invoke($res); $query['ast'] = $query_compiler->parse($string)->dump(); $query['query_main'] = $recordQuery; $query['query'] = $params['body']; $query['query_string'] = json_encode($params['body']); return new SearchEngineResult($results, json_encode($query), $res['took'], $offset, $res['hits']['total'], $res['hits']['total'], null, null, $facets->getAsSuggestions(), [], $this->indexName, $facets); }
public function createIndex($settings = null, $mappings = null) { // Assemble new index name $nextIndexNumber = 1; $lastIndexName = $this->getLastIndexName(); if (null != $lastIndexName) { $lastIndexNameArr = explode('_', $lastIndexName); $nextIndexNumber = array_pop($lastIndexNameArr) + 1; } $nextIndexName = $this->getIndexName() . '_' . date('Y') . '_' . $nextIndexNumber; // Create new index $params['index'] = $nextIndexName; if (null != $settings) { $params['body']['settings'] = $settings; } if (null != $mappings) { $params['body']['mappings'] = $mappings; } $this->client->indices()->create($params); // Update aliases $params = []; $params['body'] = ['actions' => [['add' => ['index' => $nextIndexName, 'alias' => $this->getIndexNameCurrent()]], ['add' => ['index' => $nextIndexName, 'alias' => $this->getIndexName()]]]]; if (null != $lastIndexName) { array_unshift($params['body']['actions'], ['remove' => ['index' => $lastIndexName, 'alias' => $this->getIndexNameCurrent()]]); } $this->client->indices()->updateAliases($params); return $nextIndexName; }
/** * @param string $snsid * @param int $status * * @return array */ protected function updateUserStatus($snsid, $status) { assert(is_numeric($status)); $params = ['index' => $this->index, 'type' => $this->type, 'id' => $snsid, 'body' => '{"doc": {"status": "' . $status . '"}}']; try { $ret = $this->client->update($params); return ['snsid' => $snsid, 'version' => $ret['_version']]; } catch (\Exception $e) { $errMsg = $e->getMessage(); $decodedArray = json_decode($errMsg, true); if (is_array($decodedArray)) { return ['snsid' => $snsid, 'error' => $decodedArray['status']]; } return ['snsid' => $snsid, 'error' => $errMsg]; } }
public function get($jobId) { $indices = $this->index->getIndices(); $docs = []; foreach ($indices as $index) { $docs[] = ['_index' => $index, '_type' => 'jobs', '_id' => $jobId]; } $i = 0; $prevException = null; while ($i < 5) { try { $result = $this->client->mget(['body' => ['docs' => $docs]]); foreach ($result['docs'] as $doc) { if ($doc['found']) { return new Job($this->configEncryptor, $doc['_source'], $doc['_index'], $doc['_type'], $doc['_version']); } } } catch (ServerErrorResponseException $e) { // ES server error, try again $this->log('error', 'Elastic server error response', ['attemptNo' => $i, 'jobId' => $jobId, 'exception' => $e]); $prevException = $e; } sleep(1 + intval(pow(2, $i) / 2)); $i++; } $this->log('alert', sprintf("Error getting job id '%s'", $jobId), ['exception' => $prevException]); return null; }
/** * @iterations 1000 * @group large */ public function asyncLarge() { $asyncDoc = $this->largeDocument; $asyncDoc['client']['future'] = 'lazy'; $response = $this->client->index($asyncDoc); $response = $response['body']['created']; }