protected function _execute(array $params)
{
$index = ElasticSearch::instance()->get_index('mg');
$index->create(array('number_of_shards' => 4, 'number_of_replicas' => 1, 'analysis' => array('analyzer' => array('indexAnalyzer' => array('type' => 'custom', 'tokenizer' => 'standard', 'filter' => array('lowercase', 'mySnowball')), 'searchAnalyzer' => array('type' => 'custom', 'tokenizer' => 'standard', 'filter' => array('standard', 'lowercase', 'mySnowball'))), 'filter' => array('mySnowball' => array('type' => 'snowball', 'language' => 'English')))), true);
Minion_CLI::write('Search index setup complete!');
}
public function action_index()
{
$this->view = new View_Search();
if ($this->request->query('query') !== NULL) {
$query = $this->request->query('query');
$queryString = new \Elastica\Query\QueryString();
$queryString->setDefaultOperator('AND');
$queryString->setQuery($query);
$elasticaQuery = new \Elastica\Query();
$elasticaQuery->setQuery($queryString);
$resultSet = ElasticSearch::instance()->search($elasticaQuery);
$results = $resultSet->getResults();
/** @var Kostache $renderer */
$renderer = Kostache::factory();
$out = array();
foreach ($results as $result) {
// Attempt to create the avatar instance.
try {
$refl = new ReflectionClass('View_Search_' . ucfirst($result->getType()));
$view = $refl->newInstance();
$view->data = $result->getData();
$out[] = $renderer->render($view);
} catch (ReflectionException $ex) {
Kohana::$log->add(LOG::ERROR, 'No search view class found for search type ":type".', array(':type' => $result->getType()));
}
}
$this->view->query = $query;
$this->view->results = $out;
}
}
public function view()
{
$section_handle = (string) $this->context[0];
$page = isset($this->context[1]) ? (int) $this->context[1] : 1;
if (empty($section_handle)) {
die('Invalid section handle');
}
$config = (object) Symphony::Configuration()->get('elasticsearch');
ElasticSearch::init();
$type = ElasticSearch::getTypeByHandle($section_handle);
if ($page === 1) {
// delete all documents in this index
$query = new Elastica_Query(array('query' => array('match_all' => array())));
$type->type->deleteByQuery($query);
}
// get new entries
$em = new EntryManager(Symphony::Engine());
$entries = $em->fetchByPage($page, $type->section->get('id'), (int) $config->{'reindex-batch-size'}, NULL, NULL, FALSE, FALSE, TRUE);
foreach ($entries['records'] as $entry) {
ElasticSearch::indexEntry($entry, $type->section);
}
$entries['total-entries'] = 0;
// last page, count how many entries in the index
if ($entries['remaining-pages'] == 0) {
// wait a few seconds, allow HTTP requests to complete...
sleep(5);
$entries['total-entries'] = $type->type->count();
}
header('Content-type: application/json');
echo json_encode(array('pagination' => array('total-pages' => (int) $entries['total-pages'], 'total-entries' => (int) $entries['total-entries'], 'remaining-pages' => (int) $entries['remaining-pages'], 'next-page' => $page + 1)));
exit;
}
/**
* Deletes a single record while ignoring relationships.
*
* @chainable
* @throws Kohana_Exception
* @return ORM
*/
public function delete()
{
if ($this->_search_enabled) {
// Delete the document from the search server.
ElasticSearch::instance()->delete_document($this->_search_type(), $this->id);
}
return parent::delete();
}
public function storeDoc($url, $esDoc)
{
Log::info("Store {$url}");
$esDoc->status->processed = gmdate('c');
$r = new Elasticsearch_Request($GLOBALS['phinde']['elasticsearch'] . 'document/' . ElasticSearch::getDocId($url), \HTTP_Request2::METHOD_PUT);
$r->setBody(json_encode($esDoc));
$r->send();
}
/**
* @return \Elastica\Type\Mapping
*/
public function send_search_mapping()
{
$type = ElasticSearch::instance()->get_type($this->_search_type());
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($type);
// Set mapping
$mapping->setProperties(array('id' => array('type' => 'integer', 'include_in_all' => FALSE), 'content' => array('type' => 'string', 'include_in_all' => TRUE)));
// Send mapping to type
$mapping->send();
}
public function view()
{
parent::view(FALSE);
if (isset($this->mode)) {
$section = $this->mode;
header('Content-Type: application/json');
echo file_get_contents(WORKSPACE . '/elasticsearch/mappings/' . $section . '.json');
die;
}
$this->addScriptToHead(URL . '/extensions/elasticsearch/assets/elasticsearch.mappings.js', 101);
$this->addStylesheetToHead(URL . '/extensions/elasticsearch/assets/elasticsearch.mappings.css', 'screen', 102);
$this->setPageType('table');
$this->setTitle(__('Symphony') . ' – ' . __('ElasticSearch') . ' – ' . __('Mappings'));
$this->appendSubheading(__('Mappings'));
$types = ElasticSearch::getAllTypes();
$tableHead = array();
$tableBody = array();
$tableHead[] = array(__('Section'), 'col');
$tableHead[] = array(__('Mapped Fields'), 'col');
$tableHead[] = array(__('Mapping JSON'), 'col');
$tableHead[] = array(__('Entries'), 'col');
if (!is_array($types) or empty($types)) {
$tableBody = array(Widget::TableRow(array(Widget::TableData(__('None Found.'), 'inactive', null, count($tableHead)))));
} else {
foreach ($types as $type) {
$col_name = Widget::TableData($type->section->get('name'));
$col_name->appendChild(Widget::Input('items[' . $type->section->get('handle') . ']', NULL, 'checkbox'));
$col_fields = Widget::TableData(implode(', ', $type->fields));
$col_json = Widget::TableData(sprintf('<a href="%s">%s.json</a>', $type->section->get('handle'), $type->section->get('handle')));
if ($type->type) {
$count = $type->type->count();
$col_count = Widget::TableData('<span id="reindex-' . $type->section->get('handle') . '">' . (string) $count . ' ' . ($count == 1 ? 'entry' : 'entries') . '</span>', $count_class . ' count-column');
} else {
$col_count = Widget::TableData('Rebuild mapping before continuing', $count_class . ' count-column inactive');
}
$attributes = array('data-handle' => $type->section->get('handle'));
if (in_array($type->section->get('handle'), $this->reindex) && $type->type) {
$attributes['data-reindex'] = 'yes';
}
$tableBody[] = Widget::TableRow(array($col_name, $col_fields, $col_json, $col_count), NULL, NULL, NULL, $attributes);
}
}
$table = Widget::Table(Widget::TableHead($tableHead), null, Widget::TableBody($tableBody), 'selectable');
$this->Form->appendChild($table);
$actions = new XMLElement('div');
$actions->setAttribute('class', 'actions');
$options = array(array(null, false, __('With Selected...')), array('reindex', false, __('Reindex Entries')), array('rebuild', false, __('Rebuild Mapping')), array('delete', false, __('Delete Mapping')));
$actions->appendChild(Widget::Apply($options));
$this->Form->appendChild($actions);
}
protected function _execute(array $params)
{
$model = $params['model'];
$limit = $params['limit'];
$times = $params['times'];
$offset = $params['offset'];
$elasticsearch = ElasticSearch::instance();
$orm = ORM::factory($model);
$type = $orm->_search_type();
$total = $orm->count_all();
// Write the header
Minion_CLI::write('#########################################' . str_repeat('#', strlen($model)) . '##');
Minion_CLI::write('# Bulk updating search indexes in model: ' . Minion_CLI::color($model, 'blue') . ' #');
Minion_CLI::write('#########################################' . str_repeat('#', strlen($model)) . '##');
Minion_CLI::write();
Minion_CLI::write('Importing ' . $limit . ' items at a time, beginning at offset ' . $offset . ', ' . $times . ' times');
Minion_CLI::write();
while ($times !== 0) {
$results = ORM::factory($model)->limit($limit)->offset($offset)->find_all();
// Break the loop if there are no more results in the database.
if ($results->count() <= 0) {
break;
}
$documents = array();
foreach ($results as $result) {
$documents[] = $result->get_search_document();
}
$elasticsearch->add_documents($type, $documents);
$current = $offset + $results->count();
Minion_CLI::write('Imported: ' . $current . '/' . $total);
$offset += $limit;
$times -= 1;
}
// Write the footer
Minion_CLI::write();
Minion_CLI::write('##########################');
Minion_CLI::write('# Bulk update completed! #');
Minion_CLI::write('##########################');
}
<?php
require_once 'elastic.php';
require_once 'redis.php';
$esObject = new ElasticSearch($index, $type, $size);
$redisHashObject = new redisHashClass($hashmap);
$doc = $redisHashObject->getDetailsById($_GET['id']);
if ($doc == NULL) {
//then query ES and store document in redis as well
$results = $esObject->getDetailsById($_GET['id']);
$doc = $results['_source'];
$redisHashObject->setDetailsById($_GET['id'], $doc);
}
$title = $doc['title'];
//$prod_desc = $doc['prod_desc'];
$link_from_fk = $doc['link_from_fk'];
$link_for_least_price = $link_from_fk;
$price_from_fk = $doc['price_from_fk'];
$least_price = $doc['least_price'];
if (isset($doc['price_from_am'])) {
$price_from_am = $doc['price_from_am'];
$img_src = $doc['image_src'];
if ($least_price == $price_from_am) {
$link_for_least_price = $doc['link_from_am'];
}
}
<?php
use Doctrine\Tests\Models\Comments\User;
use Doctrine\Tests\Models\Comments\Email;
use Doctrine\Tests\Models\Comments\Comment;
require_once __DIR__ . '/vendor/autoload.php';
require_once __DIR__ . '/ElasticSearch.php';
$sm = ElasticSearch::get();
$client = $sm->getClient();
$metadatas = $sm->getMetadataFactory()->getAllMetadata();
// Delete indexes
foreach ($metadatas as $metadata) {
if ($client->getIndex($metadata->index)->exists()) {
$client->deleteIndex($metadata->index);
}
}
// Recreate indexes and types
foreach ($metadatas as $metadata) {
if (!$client->getIndex($metadata->index)->exists()) {
$client->createIndex($metadata->index);
}
$client->createType($metadata);
}
//Install fixtures here... can use Doctrine/data-fixtures package with
//special SearchManager adapter if required.
$user1 = new User();
$user1->setName('Hash');
$user1->setUsername('mrhash');
$user1->addEmail(new Email('*****@*****.**'));
$user2 = new User();
$user2->setName('Timothy Leary');
<?php
require_once 'elastic.php';
$esObject = new ElasticSearch($index, $type, $size);
//query Elastic Search
if (isset($_POST['home_page_submit'])) {
//means home.php form is submitted
$query = $_POST['query'];
$filter = 0;
$cur_page = 0;
} else {
//filtered form is submitted
$query = $_POST['home_page_query'];
$cur_page = $_POST['cur_page'];
$filter = [];
if (!empty($_POST['brand'])) {
$terms_array = ["terms" => ["brand" => $_POST['brand']]];
$filter[] = $terms_array;
}
if (isset($_POST['range'])) {
$range = $_POST['range'];
$arr = explode('-', $_POST['range']);
$lower_bound = $arr[0];
$upper_bound = $arr[1];
if ($lower_bound == 20000) {
$upper_bound += 100000;
}
$range_array = ["range" => ["least_price" => ["lte" => $upper_bound, "gt" => $lower_bound]]];
$filter[] = $range_array;
}
if (empty($filter)) {
public function grab(&$param_pool = NULL)
{
$config = (object) Symphony::Configuration()->get('elasticsearch');
// build an object of runtime parameters
$params = (object) array('keywords' => isset($_GET['keywords']) ? $_GET['keywords'] : '', 'current-page' => isset($_GET['page']) && is_numeric($_GET['page']) ? (int) $_GET['page'] : 1, 'per-page' => isset($_GET['per-page']) && is_numeric($_GET['per-page']) ? (int) $_GET['per-page'] : $config->{'per-page'}, 'sort' => isset($_GET['sort']) ? $_GET['sort'] : $config->sort, 'direction' => isset($_GET['direction']) && in_array($_GET['direction'], array('asc', 'desc')) ? $_GET['direction'] : $config->direction, 'sections' => isset($_GET['sections']) && !empty($_GET['sections']) ? array_map('trim', explode(',', $_GET['sections'])) : NULL, 'default-sections' => !empty($config->{'default-sections'}) ? explode(',', $config->{'default-sections'}) : NULL, 'language' => isset($_GET['language']) && !empty($_GET['language']) ? array_map('trim', explode(',', $_GET['language'])) : NULL, 'default-language' => !empty($config->{'default-language'}) ? explode(',', $config->{'default-language'}) : NULL);
$params->{'keywords-raw'} = $params->keywords;
$params->keywords = ElasticSearch::filterKeywords($params->keywords);
// don't run search if not searching for anything
if (empty($params->keywords)) {
return;
}
// check valid page number
if ($params->{'current-page'} < 1) {
$params->{'current-page'} = 1;
}
// if no language passed but there are defaults, use the defaults
if ($params->{'language'} === NULL && count($params->{'default-language'})) {
$params->{'language'} = $params->{'default-language'};
}
// include this extension's own library
ElasticSearch::init();
// a query_string search type in ES accepts common (Lucene) search syntax such as
// prefixing terms with +/- and surrounding exact phrases with quotes
$query_querystring = new Elastica_Query_QueryString();
// all terms are required
$query_querystring->setDefaultOperator('AND');
// pass in keywords
$query_querystring->setQueryString($params->keywords);
// only apply the search to fields mapped as multi-type with a sub-type named "symphony_fulltext"
// this allows us to exclude fields from this generic full-site search but search them elsewhere
if ($params->{'language'}) {
$fields = array();
foreach ($params->{'language'} as $language) {
$fields[] = '*_' . $language . '.symphony_fulltext';
}
$query_querystring->setFields($fields);
} else {
$query_querystring->setFields(array('*.symphony_fulltext'));
}
// create the parent query object (a factory) into which the query_string is passed
$query = new Elastica_Query($query_querystring);
$query->setLimit($params->{'per-page'});
// TODO: check this. should it be + 1?
$query->setFrom($params->{'per-page'} * ($params->{'current-page'} - 1));
$query->setSort(array($params->{'sort'} => $params->{'direction'}));
// build a search object, this wraps an Elastica_Client and handles requests to and from the ElasticSearch server
$search = new Elastica_Search(ElasticSearch::getClient());
// search on our site index only (in case the server is running multiple indexes)
$search->addIndex(ElasticSearch::getIndex());
// create a new facet on the entry _type (section handle). this will return a list
// of sections in which the matching entries reside, and a count of matches in each
$facet = new Elastica_Facet_Terms('filtered-sections');
$facet->setField('_type');
$query->addFacet($facet);
// we also want a list of _all_ sections and their total entry counts. facets run within the context
// of the query they are attached to, so we want a new query that searches within the specified sections
// but doesn't search on the keywords (so it finds everything). ES supports this with a match_all query
// which Elastica creates by default when you create a plain query object
$query_all = new Elastica_Query();
$facet = new Elastica_Facet_Terms('all-sections');
$facet->setField('_type');
$query_all->addFacet($facet);
// build an array of all valid section handles that have mappings
$all_mapped_sections = array();
$section_full_names = array();
foreach (ElasticSearch::getAllTypes() as $type) {
// if using default config sections, check that the type exists in the default
if (count($params->{'default-sections'}) > 0 && !in_array($type->section->get('handle'), $params->{'default-sections'})) {
continue;
}
$all_mapped_sections[] = $type->section->get('handle');
// cache an array of section names indexed by their handles, quick lookup later
$section_full_names[$type->section->get('handle')] = $type->section->get('name');
}
$sections = array();
// no specified sections were sent in the params, so default to all available sections
if ($params->sections === NULL) {
$sections = $all_mapped_sections;
} else {
foreach ($params->sections as $handle) {
if (!in_array($handle, $all_mapped_sections)) {
continue;
}
$sections[] = $handle;
}
}
// a filter is an additional set of filtering that can be added to a query. filters are run
// after the query has executed, so run over the resultset and remove documents that don't
// match the criteria. they are fast and are cached by ES. we want to restrict the search
// results to within the specified sections only, so we add a filter on the _type (section handle)
// field. the filter is of type "terms" (an array of exact-match strings)
$filter = new Elastica_Filter_Terms('_type');
// build an array of field handles which should be highlighted in search results, used for building
// the excerpt on results pages. a field is marked as highlightable by giving it a "symphony_fulltext"
// field in the section mappings
$highlights = array();
// iterate over each valid section, adding it as a filter and finding any highlighted fields within
foreach ($sections as $section) {
// add these sections to the entry search
$filter->addTerm($section);
// read the section's mapping JSON from disk
$mapping = json_decode(ElasticSearch::getTypeByHandle($section)->mapping_json, FALSE);
// find fields that have symphony_highlight
foreach ($mapping->{$section}->properties as $field => $properties) {
if (!$properties->fields->symphony_fulltext) {
continue;
}
$highlights[] = array($field => (object) array());
}
}
// add the section filter to both queries (keyword search and the all entries facet search)
$query->setFilter($filter);
$query_all->setFilter($filter);
// configure highlighting for the keyword search
$query->setHighlight(array('fields' => $highlights, 'encoder' => 'html', 'fragment_size' => $config->{'highlight-fragment-size'}, 'number_of_fragments' => $config->{'highlight-per-field'}, 'pre_tags' => array('<strong class="highlight">'), 'post_tags' => array('</strong>')));
// run both queries!
$query_result = $search->search($query);
$query_all_result = $search->search($query_all);
// build root XMK element
$xml = new XMLElement($this->dsParamROOTELEMENT, NULL, array('took' => $query_result->getResponse()->getEngineTime() . 'ms', 'max-score' => round($query_result->getMaxScore(), 4)));
// append keywords to the XML
$xml_keywords = new XMLElement('keywords');
$xml_keywords->appendChild(new XMLElement('raw', General::sanitize($params->{'keywords-raw'})));
$xml_keywords->appendChild(new XMLElement('filtered', General::sanitize($params->{'keywords'})));
$xml->appendChild($xml_keywords);
// build pagination
$xml->appendChild(General::buildPaginationElement($query_result->getTotalHits(), ceil($query_result->getTotalHits() * (1 / $params->{'per-page'})), $params->{'per-page'}, $params->{'current-page'}));
// build facets
$xml_facets = new XMLElement('facets');
// merge the facets from both queries so they appear as one
$facets = array_merge($query_result->getFacets(), $query_all_result->getFacets());
foreach ($facets as $handle => $facet) {
$xml_facet = new XMLElement('facet', NULL, array('handle' => $handle));
foreach ($facet['terms'] as $term) {
// only show sections that are in default config, if it is being used
if (!in_array($term['term'], $all_mapped_sections)) {
continue;
}
$xml_facet_term = new XMLElement('term', $section_full_names[$term['term']], array('handle' => $term['term'], 'entries' => $term['count'], 'active' => in_array($term['term'], $sections) ? 'yes' : 'no'));
$xml_facet->appendChild($xml_facet_term);
}
$xml_facets->appendChild($xml_facet);
}
$xml->appendChild($xml_facets);
// if each entry is to have its full XML built and appended to the result,
// create a new EntryManager for using later on
if ($config->{'build-entry-xml'} === 'yes') {
$em = new EntryManager(Frontend::instance());
$field_pool = array();
}
// append entries
$xml_entries = new XMLElement('entries');
foreach ($query_result->getResults() as $data) {
$entry = new XMLElement('entry', NULL, array('id' => $data->getId(), 'section' => $data->getType(), 'score' => is_array($data->getScore()) ? reset($data->getScore()) : round($data->getScore(), 4)));
// append field highlights
foreach ($data->getHighlights() as $field => $highlight) {
foreach ($highlight as $html) {
$entry->appendChild(new XMLElement('highlight', $html, array('field' => $field)));
}
}
// build and append entry data
// this was pinched from Symphony's datasource class
if ($config->{'build-entry-xml'} === 'yes') {
$e = reset($em->fetch($data->getId()));
$field_data = $e->getData();
foreach ($field_data as $field_id => $values) {
if (!isset($field_pool[$field_id]) || !is_object($field_pool[$field_id])) {
$field_pool[$field_id] = FieldManager::fetch($field_id);
}
$field_pool[$field_id]->appendFormattedElement($entry, $values, FALSE, NULL, $e->get('id'));
}
}
$xml_entries->appendChild($entry);
// put each entry ID into the param pool for chaining
$param_pool['ds-elasticsearch'][] = $data->getId();
}
$xml->appendChild($xml_entries);
// log query if logging is enabled
if ($config->{'log-searches'} === 'yes') {
ElasticSearchLogs::save($params->keywords, $params->{'keywords-raw'}, $sections, $params->{'current-page'}, $query_result->getTotalHits());
}
return $xml;
}
<?php /** * Build out the matrix collection in ElasticSearch * containing all the interactions in quick searchable/sortable combination */ require_once __DIR__ . '/vendor/autoload.php'; require_once __DIR__ . '/Config.php'; require_once __DIR__ . '/classes/ElasticSearch.php'; //115451 $es = new ElasticSearch(); $es->initializeInteractionsIndex(); $es->buildInteractionIndexByAll();
public function grab(&$param_pool = NULL)
{
$config = (object) Symphony::Configuration()->get('elasticsearch');
// build an object of runtime parameters
$params = (object) array('keywords' => isset($_GET['keywords']) ? trim($_GET['keywords']) : '', 'per-page' => isset($_GET['per-page']) ? $_GET['per-page'] : $config->{'per-page'}, 'sections' => isset($_GET['sections']) ? array_map('trim', explode(',', $_GET['sections'])) : NULL, 'default-sections' => !empty($config->{'default-sections'}) ? explode(',', $config->{'default-sections'}) : NULL, 'language' => isset($_GET['language']) && !empty($_GET['language']) ? array_map('trim', explode(',', $_GET['language'])) : NULL, 'default-language' => !empty($config->{'default-language'}) ? explode(',', $config->{'default-language'}) : NULL);
$params->keywords = ElasticSearch::filterKeywords($params->keywords);
if (empty($params->keywords)) {
return;
}
// add trailing wildcard if it's not already there
if (end(str_split($params->keywords)) !== '*') {
$params->keywords = $params->keywords . '*';
}
// if no language passed but there are defaults, use the defaults
if ($params->{'language'} === NULL && count($params->{'default-language'})) {
$params->{'language'} = $params->{'default-language'};
}
ElasticSearch::init();
$query_querystring = new Elastica_Query_QueryString();
$query_querystring->setDefaultOperator('AND');
$query_querystring->setQueryString($params->keywords);
if ($params->{'language'}) {
$fields = array();
foreach ($params->{'language'} as $language) {
$fields[] = '*_' . $language . '.symphony_fulltext';
}
$query_querystring->setFields($fields);
} else {
$query_querystring->setFields(array('*.symphony_fulltext'));
}
$query = new Elastica_Query($query_querystring);
// returns loads. let's say we search for "romeo" and there are hundreds of lines that contain
// romeo but also the play title "romeo and juliet", the first 10 or 20 results might just be script lines
// containing "romeo", so the play title will not be included. so return a big chunk of hits to give a
// better chance of more different terms being in the result. a tradeoff of speed/hackiness over usefulness.
$query->setLimit(1000);
$search = new Elastica_Search(ElasticSearch::getClient());
$search->addIndex(ElasticSearch::getIndex());
$filter = new Elastica_Filter_Terms('_type');
// build an array of all valid section handles that have mappings
$all_mapped_sections = array();
$section_full_names = array();
foreach (ElasticSearch::getAllTypes() as $type) {
if (count($params->{'default-sections'}) > 0 && !in_array($type->section->get('handle'), $params->{'default-sections'})) {
continue;
}
$all_mapped_sections[] = $type->section->get('handle');
// cache an array of section names indexed by their handles, quick lookup later
$section_full_names[$type->section->get('handle')] = $type->section->get('name');
}
$sections = array();
// no specified sections were sent in the params, so default to all available sections
if ($params->sections === NULL) {
$sections = $all_mapped_sections;
} else {
foreach ($params->sections as $handle) {
if (!in_array($handle, $all_mapped_sections)) {
continue;
}
$sections[] = $handle;
}
}
//$autocomplete_fields = array();
$highlights = array();
foreach ($sections as $section) {
$filter->addTerm($section);
$mapping = json_decode(ElasticSearch::getTypeByHandle($section)->mapping_json, FALSE);
// find fields that have symphony_highlight
foreach ($mapping->{$section}->properties as $field => $properties) {
if (!$properties->fields->symphony_autocomplete) {
continue;
}
//$autocomplete_fields[] = $field;
$highlights[] = array($field => (object) array());
}
}
//$autocomplete_fields = array_unique($autocomplete_fields);
$query->setFilter($filter);
$query->setHighlight(array('fields' => $highlights, 'encoder' => 'html', 'fragment_size' => 100, 'number_of_fragments' => 3, 'pre_tags' => array('<strong>'), 'post_tags' => array('</strong>')));
// run the entry search
$entries_result = $search->search($query);
$xml = new XMLElement($this->dsParamROOTELEMENT, NULL, array('took' => $entries_result->getResponse()->getEngineTime() . 'ms'));
$words = array();
foreach ($entries_result->getResults() as $data) {
foreach ($data->getHighlights() as $field => $highlight) {
foreach ($highlight as $html) {
$words[] = $html;
}
}
}
$words = array_unique($words);
$xml_words = new XMLElement('words');
foreach ($words as $word) {
$raw = General::sanitize(strip_tags($word));
$highlighted = General::sanitize($word);
$xml_word = new XMLElement('word');
$xml_word->appendChild(new XMLElement('raw', $raw));
$xml_word->appendChild(new XMLElement('highlighted', $highlighted));
$xml_words->appendChild($xml_word);
}
$xml->appendChild($xml_words);
return $xml;
}
public static function getAllTypes()
{
self::init();
if (count(self::$types) > 0) {
return self::$types;
}
$sm = new SectionManager(Symphony::Engine());
$get_mappings = self::getIndex()->request('_mapping', Elastica_Request::GET, array());
$all_mappings = $get_mappings->getData();
self::$mappings = reset($all_mappings);
$types = array();
foreach ($sm->fetch() as $section) {
$elasticsearch_mapping_file = sprintf('%s/elasticsearch/mappings/%s.json', WORKSPACE, preg_replace('/-/', '_', $section->get('handle')));
$symphony_mapping_file = sprintf('%s/elasticsearch/mappings/%s.php', WORKSPACE, preg_replace('/-/', '_', $section->get('handle')));
// no mapping, no valid type
if (!file_exists($elasticsearch_mapping_file)) {
continue;
}
require_once $symphony_mapping_file;
$symphony_mapping_classname = sprintf('elasticsearch_%s', preg_replace('/-/', '_', $section->get('handle')));
$symphony_mapping_class = new $symphony_mapping_classname();
$elasticsearch_mapping_json = file_get_contents($elasticsearch_mapping_file);
$elasticsearch_mapping = json_decode($elasticsearch_mapping_json, FALSE);
$mapped_fields = $elasticsearch_mapping->{$section->get('handle')}->properties;
// invalid JSON
if (!$mapped_fields) {
throw new Exception('Invalid mapping JSON for ' . $section->get('handle'));
}
$fields = array();
foreach ($mapped_fields as $field => $mapping) {
$fields[] = $field;
}
$type = self::getIndex()->getType($section->get('handle'));
if (!isset(self::$mappings[$section->get('handle')])) {
$type = NULL;
}
$types[$section->get('handle')] = (object) array('section' => $section, 'fields' => $fields, 'type' => $type, 'mapping_json' => $elasticsearch_mapping_json, 'mapping_class' => $symphony_mapping_class);
}
self::$types = $types;
return $types;
}
$res_arr = $results['hits']['hits'];
$brands_arr = [];
foreach ($res_arr as $item) {
$brands_arr[] = $item['_source']['brnd'];
}
$brands_arr = array_count_values($brands_arr);
arsort($brands_arr);
return $brands_arr;
}
public static function getAllRanges($results)
{
$res_arr = $results['hits']['hits'];
$range = [0, 0, 0];
//range[0] contains no. of products between 0-10000 & same for range[1] & range[2]
foreach ($res_arr as $item) {
if (intval($item['_source']['price']) <= 10000) {
$range[0] += 1;
} else {
if (intval($item['_source']['price']) <= 20000) {
$range[1] += 1;
} else {
$range[2] += 1;
}
}
}
return $range;
}
}
$esObject = new ElasticSearch($index, $type, $size);
$results = $esObject->Query('', 0);
print_r($results['aggregations']['price_ranges']['buckets']);
public function savePreferences($context)
{
$settings = array_map('trim', $context['settings']['elasticsearch']);
$index_to_delete = NULL;
$index_to_create = NULL;
// index name has changed, so delete the original and create new
if ($settings['index-name'] !== $settings['index_name_original']) {
$index_to_delete = $settings['index_name_original'];
$index_to_create = $settings['index-name'];
}
// only try to delete existing index if it exists (i.e. don't run this
// the first time the extension is configured and there's no existing index)
if (!empty($index_to_delete)) {
// instantiate extension's ES helper class
ElasticSearch::init($settings['host'], $index_to_delete, $settings['username'], $settings['password']);
// delete original index
$index = ElasticSearch::getClient()->getIndex($index_to_delete);
if ($index->exists()) {
$index->delete();
}
// reset
ElasticSearch::flush();
}
if (!empty($index_to_create)) {
// instantiate extension's ES helper class
ElasticSearch::init($settings['host'], $index_to_create, $settings['username'], $settings['password']);
// create new index
$index = ElasticSearch::getClient()->getIndex($index_to_create);
$index_settings_file = WORKSPACE . '/elasticsearch/index.json';
$index_settings = array();
if (file_exists($index_settings_file)) {
$index_settings = json_decode(file_get_contents($index_settings_file), TRUE);
} else {
$index_settings = array();
}
if (is_null($index_settings)) {
throw new Exception('ElasticSearch: workspace/elasticsearch/index.json is not valid JSON');
}
$index->create($index_settings, TRUE);
}
unset($context['settings']['elasticsearch']['index_name_original']);
}
<?php
require_once 'elastic.php';
$esObject = new ElasticSearch($index, $type, $size);
//$jsonString = file_get_contents('scraped_amazon.json');
//$documents = json_decode($jsonString, true);
//foreach ($documents as $document) {
// $esObject->getDocIdForMerging($document);
//}
$sourceFileHandle = fopen('scraped_amazon.json', 'r') or die('unable to open source file');
while (($line = fgets($sourceFileHandle)) !== false) {
$line = substr($line, 0, -2);
if ($line[0] == '[') {
$line = str_replace('[', '', $line);
}
$document = json_decode($line, true);
$esObject->getDocIdForMerging($document);
}
?>
