public function getQueryForSearch(ArticleSearch $articleSearch)
{
// we create a query to return all the articles
// but if the criteria title is specified, we use it
if ($articleSearch->getTitle() !== null && $articleSearch != '') {
$query = new \Elastica\Query\Match();
$query->setFieldQuery('article.title', $articleSearch->getTitle());
$query->setFieldFuzziness('article.title', 0.7);
$query->setFieldMinimumShouldMatch('article.title', '80%');
} else {
$query = new \Elastica\Query\MatchAll();
}
// then we create filters depending on the chosen criterias
$boolQuery = new \Elastica\Query\Bool();
$boolQuery->addMust($query);
/*
Dates filter
We add this filter only the ispublished filter is not at "false"
*/
if ("false" != $articleSearch->isPublished() && null !== $articleSearch->getDateFrom() && null !== $articleSearch->getDateTo()) {
$boolQuery->addMust(new \Elastica\Query\Range('publishedAt', array('gte' => \Elastica\Util::convertDate($articleSearch->getDateFrom()->getTimestamp()), 'lte' => \Elastica\Util::convertDate($articleSearch->getDateTo()->getTimestamp()))));
}
// Published or not filter
if ($articleSearch->isPublished() !== null) {
$boolQuery->addMust(new \Elastica\Query\Terms('published', array($articleSearch->isPublished())));
}
$query = new \Elastica\Query($boolQuery);
$query->setSort(array($articleSearch->getSort() => array('order' => $articleSearch->getDirection())));
return $query;
}
/**
*
* @return bool
*/
protected function getTrendExtensions()
{
$es = $this->app->container->get('elastica.client');
$elasticaQueryString = new \Elastica\Query\QueryString();
$elasticaQueryString->setQuery('*');
$elasticaQuery = new \Elastica\Query();
$elasticaQuery->setQuery($elasticaQueryString);
$elasticaQuery->setSort(["stars" => 'desc']);
$elasticaIndex = $es->getIndex('packages');
$results = $elasticaIndex->search($elasticaQuery);
$hits = [];
foreach ($results->getResults() as $result) {
$hits[] = $result->getHit();
}
return $hits;
}
$elasticaTypeOfFilter->setQuery($query_collection);
$elasticaFilterAnd = new Elastica\Filter\BoolAnd();
$elasticaFilterAnd->addFilter($elasticaTypeOfFilter);
$filteredQuery = new Elastica\Query\Filtered($queryString, $elasticaFilterAnd);
$query = new Elastica\Query($filteredQuery);
}
//SORT BY
if (isset($_GET['sort-by'])) {
if (isset($_GET['sort-by-direction'])) {
$sortdirection = $_GET['sort-by-direction'];
} else {
$sortdirection = "desc";
}
$sort = array($_GET['sort-by'] => array("order" => $sortdirection));
//order by clause
$query->setSort($sort);
}
$query->setFrom($_GET['from'])->setLimit($_GET['size']);
if (isset($_GET['debug'])) {
echo '<pre>';
print_r($query);
echo '</pre>';
}
// Configure and execute the search
// Create the search object and inject the client
$resultSet = $searchThrough->search($query);
$data = array();
foreach ($resultSet as $result) {
$itemdata = $result->getData();
$itemdata['id'] = $result->getId();
$data[] = $itemdata;
public function sortBy($sort)
{
$query = new \Elastica\Query('agg_name');
$query->setSort($sort);
$this->query[] = $query;
return $query;
}
/**
* Run the search against a sanitized query string via ElasticSearch.
*
* @param string $string
* @param array $scriptProperties The scriptProperties array from the SimpleSearch snippet
* @return array
*/
public function search($string, array $scriptProperties = array())
{
$fields = $this->modx->getOption('sisea.elastic.search_fields', null, 'pagetitle^20,introtext^10,alias^5,content^1');
$fields = explode(',', $fields);
$fields = array_map('trim', $fields);
$fields = array_keys(array_flip($fields));
$fields = array_filter($fields);
if (empty($fields)) {
return false;
}
/** @var \Elastica\Query\MultiMatch $query */
$query = new \Elastica\Query\MultiMatch();
$query->setFields($fields);
$query->setQuery($string);
$customFilterScore = new \Elastica\Query\CustomFiltersScore();
$customFilterScore->setQuery($query);
$searchBoosts = $this->modx->getOption('sisea.elastic.search_boost', null, '');
$searchBoosts = explode('|', $searchBoosts);
$searchBoosts = array_map('trim', $searchBoosts);
$searchBoosts = array_keys(array_flip($searchBoosts));
$searchBoosts = array_filter($searchBoosts);
$boosts = array();
foreach ($searchBoosts as $boost) {
$arr = array('field' => '', 'value' => '', 'boost' => 1.0);
$field = explode('=', $boost);
$field = array_map('trim', $field);
$field = array_keys(array_flip($field));
$field = array_filter($field);
if (count($field) != 2) {
continue;
}
$value = explode('^', $field[1]);
$value = array_map('trim', $value);
$value = array_keys(array_flip($value));
$value = array_filter($value);
if (count($value) != 2) {
continue;
}
$arr['field'] = $field[0];
$arr['value'] = $value[0];
$arr['boost'] = $value[1];
$boosts[] = $arr;
}
if (empty($boosts)) {
$customFilterScore->addFilter(new \Elastica\Filter\Term(array('type' => 'document')), 1);
} else {
foreach ($boosts as $boost) {
$customFilterScore->addFilter(new \Elastica\Filter\Term(array($boost['field'] => $boost['value'])), $boost['boost']);
}
}
/** @var \Elastica\Query $elasticaQuery */
$elasticaQuery = new \Elastica\Query();
$elasticaQuery->setQuery($customFilterScore);
/* set limit */
$perPage = $this->modx->getOption('perPage', $scriptProperties, 10);
if (!empty($perPage)) {
$offset = $this->modx->getOption('start', $scriptProperties, 0);
$offsetIndex = $this->modx->getOption('offsetIndex', $scriptProperties, 'sisea_offset');
if (isset($_REQUEST[$offsetIndex])) {
$offset = (int) $_REQUEST[$offsetIndex];
}
$elasticaQuery->setFrom($offset);
$elasticaQuery->setSize($perPage);
}
$elasticaFilterAnd = new \Elastica\Filter\BoolAnd();
/* handle hidemenu option */
$hideMenu = $this->modx->getOption('hideMenu', $scriptProperties, 2);
if ($hideMenu != 2) {
$elasticaFilterHideMenu = new \Elastica\Filter\Term();
$elasticaFilterHideMenu->setTerm('hidemenu', $hideMenu ? 1 : 0);
$elasticaFilterAnd->addFilter($elasticaFilterHideMenu);
}
/* handle contexts */
$contexts = $this->modx->getOption('contexts', $scriptProperties, '');
$contexts = !empty($contexts) ? $contexts : $this->modx->context->get('key');
$contexts = explode(',', $contexts);
$elasticaFilterContext = new \Elastica\Filter\Terms();
$elasticaFilterContext->setTerms('context_key', $contexts);
$elasticaFilterAnd->addFilter($elasticaFilterContext);
/* handle restrict search to these IDs */
$ids = $this->modx->getOption('ids', $scriptProperties, '');
if (!empty($ids)) {
$idType = $this->modx->getOption('idType', $this->config, 'parents');
$depth = $this->modx->getOption('depth', $this->config, 10);
$ids = $this->processIds($ids, $idType, $depth);
$elasticaFilterId = new \Elastica\Filter\Term();
$elasticaFilterId->setTerm('id', $ids);
$elasticaFilterAnd->addFilter($elasticaFilterId);
}
/* handle exclude IDs from search */
$exclude = $this->modx->getOption('exclude', $scriptProperties, '');
if (!empty($exclude)) {
$exclude = $this->cleanIds($exclude);
$exclude = explode(',', $exclude);
$elasticaFilterExcludeId = new \Elastica\Filter\Term();
$elasticaFilterExcludeId->setTerm('id', $exclude);
$elasticaFilterNotId = new \Elastica\Filter\BoolNot($elasticaFilterExcludeId);
$elasticaFilterAnd->addFilter($elasticaFilterNotId);
}
/* basic always-on conditions */
$elasticaFilterPublished = new \Elastica\Filter\Term();
$elasticaFilterPublished->setTerm('published', 1);
$elasticaFilterAnd->addFilter($elasticaFilterPublished);
$elasticaFilterSearchable = new \Elastica\Filter\Term();
$elasticaFilterSearchable->setTerm('searchable', 1);
$elasticaFilterAnd->addFilter($elasticaFilterSearchable);
$elasticaFilterDeleted = new \Elastica\Filter\Term();
$elasticaFilterDeleted->setTerm('deleted', 0);
$elasticaFilterAnd->addFilter($elasticaFilterDeleted);
$elasticaQuery->setFilter($elasticaFilterAnd);
/* sorting */
if (!empty($scriptProperties['sortBy'])) {
$sortDir = $this->modx->getOption('sortDir', $scriptProperties, 'desc');
$sortDirs = explode(',', $sortDir);
$sortBys = explode(',', $scriptProperties['sortBy']);
$dir = 'desc';
$sortArray = array();
for ($i = 0; $i < count($sortBys); $i++) {
if (isset($sortDirs[$i])) {
$dir = $sortDirs[$i];
}
$sortArray[] = array($sortBys[$i] => $dir);
}
$elasticaQuery->setSort($sortArray);
}
/* prepare response array */
$response = array('total' => 0, 'start' => !empty($offset) ? $offset : 0, 'limit' => $perPage, 'status' => 0, 'query_time' => 0, 'results' => array());
$elasticaResultSet = $this->index->search($elasticaQuery);
$elasticaResults = $elasticaResultSet->getResults();
$totalResults = $elasticaResultSet->getTotalHits();
if ($totalResults > 0) {
$response['total'] = $totalResults;
$response['query_time'] = $elasticaResultSet->getTotalTime();
$response['status'] = 1;
$response['results'] = array();
foreach ($elasticaResults as $doc) {
$d = $doc->getData();
/** @var modResource $resource */
$resource = $this->modx->newObject($d['class_key']);
if ($resource->checkPolicy('list')) {
$response['results'][] = $d;
}
}
}
return $response;
}
public function getArticles($user, $searchTerm, $page, $pageOffset, $category, $orderby, $colors, $finder, $elasticIndex)
{
$boolQuery = new \Elastica\Query\BoolQuery();
if ($category != NULL) {
$query = $this->getEntityManager()->createQuery("SELECT c.id FROM OrthIndexBundle:Categories c WHERE c.id LIKE :category")->setParameter('category', $category . "%");
$queryResult = $query->getResult(\Doctrine\ORM\Query::HYDRATE_ARRAY);
foreach ($queryResult as $categoryId) {
$categoryArray[] = $categoryId['id'];
}
$categoryQuery = new \Elastica\Query\Terms();
$categoryQuery->setTerms('catRef', $categoryArray);
$boolQuery->addMust($categoryQuery);
}
if ($searchTerm) {
$fieldQuery = new \Elastica\Query\Match();
$fieldQuery->setFieldQuery('allField', $searchTerm);
$fieldQuery->setFieldOperator('allField', 'AND');
$fieldQuery->setFieldMinimumShouldMatch('allField', '70%');
$fieldQuery->setFieldFuzziness('allField', '0.8');
$fieldQuery->setFieldAnalyzer('allField', 'custom_search_analyzer');
$boolQuery->addMust($fieldQuery);
}
if ($colors != NULL) {
$colorQuery = new \Elastica\Query\Terms();
$colorQuery->setTerms('variants.variantvalues.otherTerms', $colors);
$colorNested = new \Elastica\Query\Nested('variants');
$colorNested->setPath('variants.variantvalues');
$colorNested->setQuery($colorQuery);
$boolQuery->addMust($colorNested);
}
$agg = new \Elastica\Aggregation\Terms("catRef");
$agg->setSize(5000);
$agg->setField('catRef');
$boolFilter = new \Elastica\Filter\Bool();
if ($user == "anon.") {
$boolFilter->addShould(new \Elastica\Filter\Terms('customized', array(0)));
} else {
$boolFilter->addShould(new \Elastica\Filter\Terms('customized', array(0, $user->getCustomerRef())));
}
$filtered = new \Elastica\Query\Filtered($boolQuery, $boolFilter);
$query = new \Elastica\Query();
$query->setQuery($filtered);
//if( $colors != NULL) {
// $query->setFilter($colorNested);
//}
if ($orderby == 'desc') {
$query->setSort(array('variants.price' => array('order' => 'desc')));
} elseif ($orderby == 'asc') {
$query->setSort(array('variants.price' => array('order' => 'asc')));
}
$query->addAggregation($agg);
$query->setSize(12);
$query->setFrom($pageOffset);
$articles = $finder->find($query);
$aggregations = $elasticIndex->search($query);
$result = array("articles" => $articles, "aggs" => $aggregations, "rQuery" => $query);
return $result;
}
public function searchSample($criteria, $dataset, $sorts = array(), $operator = 'OR')
{
/*if($limit <= 0) {
$limit = Yii::app()->params['es_search']['limits']['default'];
}*/
$keyword = $criteria['keyword'];
$eQueryString = new \Elastica\Query\QueryString();
$eQueryString->setDefaultOperator($operator);
$eQueryString->setQuery($keyword);
$eQuery = new \Elastica\Query();
$eQuery->setQuery($eQueryString);
//$eQuery->setFrom($offset);
//$eQuery->setLimit($limit);
$filterAnd = new \Elastica\Filter\BoolAnd();
if (isset($criteria['common_name'])) {
$filterAnd->addFilter($this->newTerms('common_name', $criteria['common_name']));
}
if (is_array($dataset) && !empty($dataset)) {
$ids = new \Elastica\Filter\Ids();
$ids->setIds($dataset);
$filterAnd->addFilter($ids);
}
$arrayAnd = $filterAnd;
$arrayAnd = $arrayAnd->toArray();
if (count($arrayAnd['and']) > 0) {
$eQuery->setPostFilter($filterAnd);
}
$sortList = array_merge(array('_score' => array('order' => 'desc')), $sorts);
$eQuery->setSort($sortList);
$index = Yii::app()->elastic->client->getIndex('gigadb');
$type = $index->getType('sample');
$eResultSet = $type->search($eQuery);
$results = $eResultSet->getResults();
$total = $eResultSet->getTotalHits();
$data = array();
foreach ($results as $result) {
if ($result) {
$data[] = $result->getData();
}
}
$re = array('data' => $data, 'total' => $total);
//Yii::log(print_r($re, true), 'debug');
return $re;
}
/**
* @SWG\Api(
* path="/posts/search",
* @SWG\Operation(
* method="POST",
* summary="Search items in posts",
* nickname="posts_search",
* @SWG\Parameter(
* name="post_type",
* required=false,
* type="string"
* ),
* @SWG\Parameter(
* name="filters",
* description="filter values, eg. [{my_parameter: 'mustmatchthis'}]
* required=false,
* type="array[]"
* ),
* @SWG\Parameter(
* name="search_phrase",
* required=false,
* type="string"
* ),
* @SWG\Parameter(
* name="sort",
* description="sort configuration, eg [{'post_name':{'order':'asc'}}]",
* required=false,
* type="array[]"
* ),
* @SWG\Parameter(
* name="limit",
* description="Max number of documents in return",
* required=false,
* type="integer"
* )
* )
* )
*/
function search()
{
$result = array();
$body = $this->getBodyAsArray();
$required_parameters = array();
$optional_parameters = array('post_type', 'filters', 'search_phrase', 'sort', 'limit', 'size', 'from');
$this->check_req_opt_param($required_parameters, $optional_parameters, $body);
$index = $this->elastica->getIndex($this->getElasticsearchIndex());
$query = new \Elastica\Query();
$boolean = new \Elastica\Query\Bool();
$added = false;
$type = isset($body['post_type']) ? $body['post_type'] : null;
if ($type) {
$q = new \Elastica\Query\Term(array('_type' => $type));
$boolean->addMust($q);
$added = true;
}
$filters = isset($body['filters']) ? (array) $body['filters'] : array();
$filters = array_filter($filters);
$filterAnd = '';
if ($filters) {
foreach ($filters as $key => $vals) {
if (!is_array($vals)) {
$vals = array($vals);
}
if (strpos(implode('', $vals), '*')) {
foreach ($vals as $val) {
$extraQuery = new \Elastica\Query\SimpleQueryString($val, array($key));
$boolean->addMust($extraQuery);
}
} else {
$filterOr = new \Elastica\Filter\BoolOr();
foreach ($vals as $val) {
$filter = new \Elastica\Filter\Term();
$filter->setTerm($key, $val);
$filterOr->addFilter($filter);
}
if (!$filterAnd) {
$filterAnd = new \Elastica\Filter\BoolAnd();
}
$filterAnd->addFilter($filterOr);
}
}
}
$search_phrase = isset($body['search_phrase']) ? $body['search_phrase'] : null;
if (isset($search_phrase) && !empty($search_phrase)) {
$word = strtolower($search_phrase) . '*';
$elasticaQueryString = new \Elastica\Query\SimpleQueryString($word);
$boolean->addMust($elasticaQueryString);
$added = true;
}
if ($added) {
$query->setQuery($boolean);
}
if ($filterAnd) {
$query->setPostFilter($filterAnd);
}
$limit = isset($body['limit']) ? (int) $body['limit'] : null;
if ($limit) {
$query->setSize($limit);
}
$sort = isset($body['sort']) ? (array) $body['sort'] : null;
if ($sort) {
$query->setSort($sort);
// example: array( 'post_date' => array( 'order' => 'desc' ) )
}
$body['from'] = isset($body['from']) && $body['from'] ? $body['from'] : 0;
$body['size'] = isset($body['size']) && $body['size'] ? $body['size'] : 1000;
$elasticaResultSet = $index->search($query, ['from' => $body['from'], 'size' => $body['size']]);
foreach ($elasticaResultSet->getResults() as $elasticaResult) {
$result[] = $elasticaResult->getData();
}
echo json_encode(array('hits' => $result, 'count' => $index->count($query)));
}
if (!empty($param['date_from']) || !empty($param['date_to'])) {
$date_param = array();
if (!empty($param['date_from'])) {
$date_param["gte"] = date("Y-m-d H:i:s", strtotime($param['date_from']));
}
if (!empty($param['date_to'])) {
$date_param["lte"] = date("Y-m-d H:i:s", strtotime($param['date_to']));
}
$date_filter = new Elastica\Filter\NumericRange("create_date", $date_param);
$filter_list[] = $date_filter;
}
if (!empty($filter_list)) {
$filter->setFilters($filter_list);
$query = new Elastica\Query\Filtered($query, $filter);
}
$query = new Elastica\Query($query);
$query->setSort(array(array('create_date' => array('order' => 'desc'))));
$query->setSize(5000);
$start_time = microtime(true);
$result_set = $chat_type->search($query);
$end_time = microtime(true);
$data = array();
while ($result = $result_set->current()) {
$d = $result->getSource();
$d['id'] = $result->getId();
$data[] = $d;
$result_set->next();
}
$response = array('count' => $result_set->count(), 'data' => $data, 'time' => $end_time - $start_time);
header("Content-Type: application/json; charset=utf-8");
echo json_encode($response);
protected function doQuery($sourceLanguage, $targetLanguage, $text)
{
/* Two query system:
* 1) Find all strings in source language that match text
* 2) Do another query for translations for those strings
*/
$connection = $this->getClient()->getConnection();
$oldTimeout = $connection->getTimeout();
$connection->setTimeout(10);
$fuzzyQuery = new \Elastica\Query\FuzzyLikeThis();
$fuzzyQuery->setLikeText($text);
$fuzzyQuery->addFields(array('content'));
$boostQuery = new \Elastica\Query\FunctionScore();
if ($this->useWikimediaExtraPlugin()) {
$boostQuery->addFunction('levenshtein_distance_score', array('text' => $text, 'field' => 'content'));
} else {
$groovyScript = <<<GROOVY
import org.apache.lucene.search.spell.*
new LevensteinDistance().getDistance(srctxt, _source['content'])
GROOVY;
$script = new \Elastica\Script($groovyScript, array('srctxt' => $text), \Elastica\Script::LANG_GROOVY);
$boostQuery->addScriptScoreFunction($script);
}
$boostQuery->setBoostMode(\Elastica\Query\FunctionScore::BOOST_MODE_REPLACE);
// Wrap the fuzzy query so it can be used as a filter.
// This is slightly faster, as ES can throw away the scores by this query.
$fuzzyFilter = new \Elastica\Filter\Query();
$fuzzyFilter->setQuery($fuzzyQuery);
$boostQuery->setFilter($fuzzyFilter);
// Use filtered query to wrap function score and language filter
$filteredQuery = new \Elastica\Query\Filtered();
$languageFilter = new \Elastica\Filter\Term();
$languageFilter->setTerm('language', $sourceLanguage);
$filteredQuery->setFilter($languageFilter);
$filteredQuery->setQuery($boostQuery);
// The whole query
$query = new \Elastica\Query();
$query->setQuery($filteredQuery);
// The interface usually displays three best candidates. These might
// come from more than three source things, if the translations are
// the same. In other words suggestions are grouped by the suggested
// translation. This algorithm might not find all suggestions, if the
// top N best matching source texts don't have equivalent translations
// in the target language, but worse matches which we did not fetch do.
// This code tries to balance between doing too many or too big queries
// and not fetching enough results to show all possible suggestions.
$sizeFirst = 100;
$sizeSecond = $sizeFirst * 5;
$query->setFrom(0);
$query->setSize($sizeFirst);
$query->setParam('_source', array('content'));
$cutoff = isset($this->config['cutoff']) ? $this->config['cutoff'] : 0.65;
$query->setParam('min_score', $cutoff);
$query->setSort(array('_score', '_uid'));
// This query is doing two unrelated things:
// 1) Collect the message contents and scores so that they can
// be accessed later for the translations we found.
// 2) Build the query string for the query that fetches the translations.
$contents = $scores = $terms = array();
do {
$resultset = $this->getType()->search($query);
if (count($resultset) === 0) {
break;
}
foreach ($resultset->getResults() as $result) {
$data = $result->getData();
$score = $result->getScore();
$sourceId = preg_replace('~/[^/]+$~', '', $result->getId());
$contents[$sourceId] = $data['content'];
$scores[$sourceId] = $score;
$terms[] = "{$sourceId}/{$targetLanguage}";
}
// Check if it looks like that we are hitting the long tail already.
// Otherwise, we'll do a query to fetch some more to reach a "sane"
// breaking point, i.e. include all suggestions with same content
// for reliable used X times statistics.
if (count(array_unique($scores)) > 5) {
break;
}
// Okay, We are now in second iteration of the loop. We already got
// lots of suggestions. We will give up for now even if it means we
// return in some sense incomplete results.
if (count($resultset) === $sizeSecond) {
break;
}
// After the first query, the smallest score is the new threshold.
$query->setParam('min_score', $score);
$query->setFrom($query->getParam('size') + $query->getParam('from'));
$query->setSize($sizeSecond);
// Break if we already got all hits
} while ($resultset->getTotalHits() > count($contents));
$suggestions = array();
// Skip second query if first query found nothing. Keeping only one return
// statement in this method to avoid forgetting to reset connection timeout
if ($terms !== array()) {
$idQuery = new \Elastica\Query\Terms();
$idQuery->setTerms('_id', $terms);
$query = new \Elastica\Query($idQuery);
$query->setSize(25);
$query->setParam('_source', array('wiki', 'uri', 'content', 'localid'));
$resultset = $this->getType()->search($query);
foreach ($resultset->getResults() as $result) {
$data = $result->getData();
// Construct the matching source id
$sourceId = preg_replace('~/[^/]+$~', '', $result->getId());
$suggestions[] = array('source' => $contents[$sourceId], 'target' => $data['content'], 'context' => $data['localid'], 'quality' => $scores[$sourceId], 'wiki' => $data['wiki'], 'location' => $data['localid'] . '/' . $targetLanguage, 'uri' => $data['uri']);
}
// Ensure reults are in quality order
uasort($suggestions, function ($a, $b) {
if ($a['quality'] === $b['quality']) {
return 0;
}
return $a['quality'] < $b['quality'] ? 1 : -1;
});
}
$connection->setTimeout($oldTimeout);
return $suggestions;
}
