private function getCountQuery($journal)
{
$filter = new Term();
$filter->setTerm('journalUsers.journal.id', $journal->getId());
$filterQuery = new Query\Filtered();
$filterQuery->setFilter($filter);
$query = new Query($filterQuery);
$query->setSize(0);
return $query;
}
/**
* search page index controller
*
* @param Request $request
* @param int $page
* @return \Symfony\Component\HttpFoundation\Response
*/
public function indexAction(Request $request, $page = 1)
{
/**
* @var \Ojs\SearchBundle\Manager\SearchManager $searchManager
*/
$searchManager = $this->get('ojs_search_manager');
$getRoles = $request->query->get('role_filters');
$getSubjects = $request->query->get('subject_filters');
$getJournals = $request->query->get('journal_filters');
$roleFilters = !empty($getRoles) ? explode(',', $getRoles) : [];
$subjectFilters = !empty($getSubjects) ? explode(',', $getSubjects) : [];
$journalFilters = !empty($getJournals) ? explode(',', $getJournals) : [];
$queryType = $request->query->has('type') ? $request->get('type') : 'basic';
$query = filter_var($request->get('q'), FILTER_SANITIZE_STRING);
$section = filter_var($request->get('section'), FILTER_SANITIZE_STRING);
$searcher = $this->get('fos_elastica.index.search');
$searchQuery = new Query('_all');
$boolQuery = new Query\Bool();
$match = new Query\Match();
$match->setField('status', 3);
$boolQuery->addShould($match);
$match = new Query\Match();
$match->setField('published', true);
$boolQuery->addShould($match);
//set query according to query type
if ($queryType == 'basic') {
$fieldQuery = new Query\MultiMatch();
$fieldQuery->setFields(['_all']);
$fieldQuery->setQuery($query);
$boolQuery->addMust($fieldQuery);
} elseif ($queryType == 'advanced') {
$parseQuery = $searchManager->parseSearchQuery($query);
foreach ($parseQuery as $searchTerm) {
$condition = $searchTerm['condition'];
$advancedFieldQuery = new Query\MultiMatch();
$advancedFieldQuery->setFields([$searchTerm['searchField']]);
$advancedFieldQuery->setQuery($searchTerm['searchText']);
if ($condition == 'AND') {
$boolQuery->addMust($advancedFieldQuery);
} elseif ($condition == 'OR') {
$boolQuery->addShould($advancedFieldQuery);
} elseif ($condition == 'NOT') {
$boolQuery->addMustNot($advancedFieldQuery);
}
}
} elseif ($queryType == 'tag') {
$matchQuery = new Query\Match();
$matchQuery->setField('tags', $query);
$boolQuery->addMust($matchQuery);
}
//set aggregations if requested
if (!empty($roleFilters) || !empty($subjectFilters) || !empty($journalFilters)) {
foreach ($roleFilters as $role) {
$match = new Query\Match();
$match->setField('user.userJournalRoles.role.name', $role);
$boolQuery->addMust($match);
}
foreach ($subjectFilters as $subject) {
$match = new Query\Match();
$match->setField('subjects', $subject);
$boolQuery->addMust($match);
}
foreach ($journalFilters as $journal) {
$match = new Query\Match();
$match->setField('user.userJournalRoles.journal.title', $journal);
$boolQuery->addMust($match);
}
}
//set our boolean query
$searchQuery->setQuery($boolQuery);
//get all result
$searchQuery->setSize(1000);
//get role aggregation
$roleAgg = new Aggregation\Terms('roles');
$roleAgg->setField('userJournalRoles.role.name');
$roleAgg->setOrder('_term', 'asc');
$roleAgg->setSize(0);
$searchQuery->addAggregation($roleAgg);
//get subject aggregation
$subjectAgg = new Aggregation\Terms('subjects');
$subjectAgg->setField('subjects');
$subjectAgg->setOrder('_term', 'asc');
$subjectAgg->setSize(0);
$searchQuery->addAggregation($subjectAgg);
//get journal aggregation
$journalAgg = new Aggregation\Terms('journals');
$journalAgg->setField('journal.raw');
$journalAgg->setOrder('_term', 'asc');
$journalAgg->setSize(0);
$searchQuery->addAggregation($journalAgg);
/**
* @var ResultSet $resultData
*/
$resultData = $searcher->search($searchQuery);
$roles = $resultData->getAggregation('roles')['buckets'];
$subjects = $resultData->getAggregation('subjects')['buckets'];
$journals = $resultData->getAggregation('journals')['buckets'];
if ($resultData->count() > 0) {
/**
* manipulate result data for easily use on template
*/
$results = $searchManager->buildResultsObject($resultData, $section);
/**
* if search section is not defined or empty redirect to first result section
*/
if (empty($section)) {
$section = array_keys($results)[0];
$redirectParams = array_merge($request->query->all(), ['section' => $section]);
return $this->redirectToRoute('ojs_search_index', $redirectParams);
} else {
/**
* if section result is empty redirect to first that have result section
*/
if (!isset($results[$section])) {
foreach ($results as $resultKey => $result) {
if ($result['total_item'] > 0) {
$redirectParams = array_merge($request->query->all(), ['section' => $resultKey]);
return $this->redirectToRoute('ojs_search_index', $redirectParams);
}
}
}
}
$adapter = new ArrayAdapter($results[$section]['data']);
$pagerfanta = new Pagerfanta($adapter);
$pagerfanta->setMaxPerPage(10);
$pagerfanta->setCurrentPage($page);
$results[$section]['data'] = $pagerfanta->getCurrentPageResults();
/**
* add search query to query history
* history data stores on session
*/
$this->addQueryToHistory($request, $query, $queryType, $resultData->count());
$data = ['results' => $results, 'query' => $query, 'queryType' => $queryType, 'section' => $section, 'total_count' => $searchManager->getTotalHit(), 'roles' => $roles, 'subjects' => $subjects, 'journals' => $journals, 'role_filters' => $roleFilters, 'subject_filters' => $subjectFilters, 'journal_filters' => $journalFilters, 'pagerfanta' => $pagerfanta];
} else {
$data = ['query' => $query, 'queryType' => $queryType, 'total_count' => $searchManager->getTotalHit(), 'journals' => []];
}
return $this->render('OjsSiteBundle:Search:index.html.twig', $data);
}
/**
* @param \Elastica\Query $query
* @param \Spryker\Client\Search\Dependency\Plugin\PaginationConfigBuilderInterface $paginationConfig
* @param array $requestParameters
*
* @return void
*/
protected function addPaginationToQuery(Query $query, PaginationConfigBuilderInterface $paginationConfig, array $requestParameters)
{
$currentPage = $paginationConfig->getCurrentPage($requestParameters);
$itemsPerPage = $paginationConfig->getCurrentItemsPerPage($requestParameters);
$query->setFrom(($currentPage - 1) * $itemsPerPage);
$query->setSize($itemsPerPage);
}
protected function getStates()
{
$results = [];
$query = new Elastica\Query();
$query->setSize(0);
$agg = new Elastica\Aggregation\Nested('states', 'state');
$st_terms = new Elastica\Aggregation\Terms('abbrev');
$st_terms->setField('state.abbrev');
$st_terms->setOrder('_term', 'asc');
$st_terms->setSize(0);
$state_terms = new Elastica\Aggregation\Terms('full');
$state_terms->setField('state.full');
$st_terms->addAggregation($state_terms);
$agg->addAggregation($st_terms);
$query->addAggregation($agg);
/* @var $elastica_client Elastica\Client */
$elastica_client = $this->getServiceLocator()->getServiceLocator()->get('elastica-client');
try {
/* @var $response \Elastica\Response */
$response = $elastica_client->request('usgeodb/locality/_search?query_cache=true', Request::GET, $query->toArray());
$data = $response->getData();
$aggregations = isset($data['aggregations']['states']['abbrev']['buckets']) ? $data['aggregations']['states']['abbrev']['buckets'] : [];
foreach ($aggregations as $aggregation) {
$key = strtoupper($aggregation['key']);
$value = ucwords($aggregation['full']['buckets'][0]['key']);
$results[$key] = $value;
}
} catch (\Exception $e) {
}
return $results;
}
public function autosuggest($text)
{
$queryString = new QueryString($text);
$elasticaQuery = new ElasticaQuery($queryString);
$elasticaQuery->setSize(10);
$result = $this->elastica->search($elasticaQuery, 'locations');
return $this->prepareSuggestResult($result);
}
/**
* @param int $offset
* @param int $size
*
* @return SearcherInterface
*/
public function setPagination($offset, $size)
{
if (is_int($offset)) {
$this->query->setFrom($offset);
}
if (is_int($size)) {
$this->query->setSize($size);
}
return $this;
}
/**
* @param $keyword
* @return \Elastica\ResultSet
*/
public function search($keyword)
{
$query = new Query();
$query->setSize(1000);
$fuzzyQuery = new Query\FuzzyLikeThis();
$fuzzyQuery->setLikeText($keyword);
$fuzzyQuery->setMinSimilarity(0.7);
$query->setQuery($fuzzyQuery);
$results = $this->search->search($query);
return $results;
}
/**
* @group unit
*/
public function testSuggestShouldNotRemoveOtherParameters()
{
$query1 = new Query();
$query2 = new Query();
$suggest = new Suggest();
$suggest->setGlobalText('test');
$query1->setSize(40);
$query1->setSuggest($suggest);
$query2->setSuggest($suggest);
$query2->setSize(40);
$this->assertEquals($query1->toArray(), $query2->toArray());
}
public function testQuerySizeOverride()
{
$query = new Query();
$query->setSize(100);
$index = $this->_createIndex('test_1');
$index->refresh();
// Waits for the index to be fully created.
$type = $index->getType('scanAndScrollTest');
$search = new Search($this->_getClient());
$search->addIndex($index)->addType($type);
$search->setQuery($query);
$scanAndScroll = new ScanAndScroll($search);
$scanAndScroll->sizePerShard = 10;
$scanAndScroll->rewind();
$this->assertEquals(10, $query->getParam('size'));
}
/**
* @param string $term text to search
* @return \Status
*/
public function searchText($term)
{
$term = trim($term);
// No searching for nothing! That takes forever!
if (!$term) {
return null;
}
$query = new Query();
$offset = min($this->offset, static::MAX_OFFSET);
if ($offset) {
$query->setFrom($offset);
}
if ($this->limit) {
$query->setSize($this->limit);
}
$filter = new Bool();
// filters
if ($this->namespaces) {
$filter->addMust(new Terms('namespace', $this->namespaces));
}
if ($this->pageIds) {
$filter->addMust(new Terms('pageid', $this->pageIds));
}
if ($this->moderationStates) {
$filter->addMust(new Terms('revisions.moderation_state', $this->moderationStates));
}
// only apply filters if there are any
if ($filter->toArray()) {
$query->setFilter($filter);
}
$sortArgs = $this->getSortArgs();
if (isset($sortArgs[$this->sort]) && $sortArgs[$this->sort]) {
$query->setSort($sortArgs[$this->sort]);
}
// @todo: interwiki stuff? (see \CirrusSearch)
$searcher = new Searcher($query, false, $this->type);
return $searcher->searchText($term);
}
/**
* @return false|\Elastica\Query
* @throws ErrorException
*/
public function getQuery()
{
if ($this->validate() === false) {
return false;
}
$termsAgg = new Terms(self::GROUP_AGGREGATION_NAME);
$termsAgg->setField($this->group['by']);
$termsAgg->setSize(self::MAX_SIZE_AGGREGATION_ITEMS);
$filterBool = new Bool();
if ($this->condition) {
$this->addCondition($filterBool);
}
$this->addAggregation($termsAgg);
$queryFiltred = new \Elastica\Query\Filtered();
$queryFiltred->setFilter($filterBool);
$queryFiltred->setQuery(new \Elastica\Query\MatchAll());
$query = new Query();
// These are not needed only aggregation
$query->setSize(0);
$query->setQuery($queryFiltred);
$query->addAggregation($termsAgg);
return $query;
}
/**
* @param Request $request
* @return JsonResponse
*/
public function autoCompleteAction(Request $request)
{
$q = filter_var($request->get('q'), FILTER_SANITIZE_STRING);
$search = $this->get('fos_elastica.index.search.journal');
$notCollectJournals = [];
if ($request->query->has('notCollectJournals')) {
$notCollectJournalsParam = $request->query->get('notCollectJournals');
if (!empty($notCollectJournalsParam) && is_array($notCollectJournalsParam)) {
$notCollectJournals = $notCollectJournalsParam;
}
}
$searchQuery = new Query('_all');
$boolQuery = new Query\BoolQuery();
$fieldQuery = new Query\MultiMatch();
$fieldQuery->setFields(['title']);
$fieldQuery->setQuery(strtoupper($q));
$fieldQuery->setFuzziness(0.7);
$boolQuery->addMust($fieldQuery);
$searchQuery->setQuery($boolQuery);
$searchQuery->setSize(10);
$results = $search->search($searchQuery);
$data = [];
foreach ($results as $result) {
if (!in_array($result->getId(), $notCollectJournals)) {
$data[] = ['id' => $result->getId(), 'text' => $result->getData()['title']];
}
}
return new JsonResponse($data);
}
public function Addons()
{
$list = Addon::get();
$search = $this->request->getVar('search');
$type = $this->request->getVar('type');
$compat = $this->request->getVar('compatibility');
$tags = $this->request->getVar('tags');
$sort = $this->request->getVar('sort');
$view = $this->request->getVar('view');
if (!$view) {
$view = 'list';
}
if (!in_array($sort, array('name', 'downloads', 'newest'))) {
$sort = null;
}
// Proxy out a search to elastic if any parameters are set.
if ($search || $type || $compat || $tags) {
$filter = new BoolAnd();
$query = new Query();
$query->setSize(count($list));
if ($search) {
$match = new Match();
$match->setField('_all', $search);
$query->setQuery($match);
}
if ($type) {
$filter->addFilter(new Term(array('type' => $type)));
}
if ($compat) {
$filter->addFilter(new Terms('compatible', (array) $compat));
}
if ($tags) {
$filter->addFilter(new Terms('tag', (array) $tags));
}
if ($type || $compat || $tags) {
$query->setFilter($filter);
}
$list = new ResultList($this->elastica->getIndex(), $query);
if ($sort) {
$ids = $list->column('ID');
if ($ids) {
$list = Addon::get()->byIDs($ids);
} else {
$list = new ArrayList();
}
}
} else {
if (!$sort) {
$sort = 'downloads';
}
}
switch ($sort) {
case 'name':
$list = $list->sort('Name');
break;
case 'newest':
$list = $list->sort('Released', 'DESC');
break;
case 'downloads':
$list = $list->sort('Downloads', 'DESC');
break;
}
$list = new PaginatedList($list, $this->request);
$list->setPageLength(16);
return $list;
}
/**
* Tag cloud page
* @return \Symfony\Component\HttpFoundation\Response
*/
public function tagCloudAction()
{
$search = $this->container->get('fos_elastica.index.search');
$searchQuery = new Query();
$searchQuery->setSize(0);
//get tags aggregation
$tagsAgg = new Aggregation\Terms('tags');
$tagsAgg->setField('tags');
$tagsAgg->setSize(500);
$searchQuery->addAggregation($tagsAgg);
/**
* @var ResultSet $results
*/
$results = $search->search($searchQuery);
$data['tags'] = [];
foreach ($results->getAggregations()['tags']['buckets'] as $result) {
$keys = array_filter(explode(',', $result['key']));
if (is_array($keys)) {
foreach ($keys as $key) {
$data['tags'][] = trim($key);
}
}
}
return $this->render('OjsSiteBundle:Search:tags_cloud.html.twig', $data);
}
/**
* @param \Elastica\Query $baseQuery
*
* @return void
*/
protected function setLimit($baseQuery)
{
if ($this->limit !== null) {
$baseQuery->setSize($this->limit);
}
}
public function attributeAction()
{
$result = ['outcome' => 0];
$entityClass = 'Lead\\Entity\\LeadAttribute';
$id = $this->getEvent()->getRouteMatch()->getParam('id', '0');
$action = $this->params()->fromQuery('action', 'update');
$em = $this->getEntityManager();
$objRepository = $em->getRepository($entityClass);
if ($id) {
switch ($action) {
case 'update':
$attributeType = $this->params()->fromQuery('type', false);
if ($attributeType) {
/* @var $entity \Lead\Entity\LeadAttribute */
$entity = $objRepository->find($id);
if ($entity && $entity instanceof $entityClass) {
$entity->setAttributeType($attributeType);
try {
$em->persist($entity);
$em->flush();
$this->createServiceEvent()->setEntityId($id)->setEntityClass($entityClass)->setDescription("Lead Attribute Edited")->setMessage("Lead Attribute: " . $entity->getAttributeDesc() . " was edited.");
$this->logEvent("EditAction.post");
$result['outcome'] = 1;
} catch (\Exception $e) {
$this->createServiceEvent()->setEntityId($id)->setEntityClass($entityClass)->setDescription("Lead Attribute Edited");
$this->logError($e);
}
}
}
break;
case 'relationship':
/* @var $entity \Lead\Entity\LeadAttribute */
$entity = $objRepository->find($id);
if ($entity && $entity instanceof $entityClass) {
$attributeType = $entity->getAttributeType();
/* @var $qb \Doctrine\ORM\QueryBuilder */
$qb = $em->createQueryBuilder();
$qb->add('select', 'e')->add('from', 'Agent\\Entity\\Relationship' . ' e')->where(' e.allowed LIKE :attributeType')->setParameter('attributeType', "%" . $attributeType . "%");
$dql = $qb->getDQL();
$query = $qb->getQuery();
$query->useQueryCache(true);
$query->useResultCache(true, 3600, 'relationship-' . md5($dql));
$result['data'] = $query->getArrayResult();
$result['outcome'] = 1;
}
break;
case 'values':
$qb = $em->createQueryBuilder()->select('v.value')->from('Lead\\Entity\\LeadAttributeValue', 'v')->where('v.attribute = :id')->setParameter('id', $id)->distinct(true);
$dql = $qb->getDQL();
$query = $qb->getQuery();
$query->useQueryCache(true);
$query->useResultCache(true, 3600, 'leadattributevalue-' . md5($dql));
$results = $query->getArrayResult();
if ($results) {
$result['data'] = $results;
$result['outcome'] = 1;
}
break;
case 'limits':
$attribute_query = new Elastica\Query\Nested();
$attribute_query->setPath('attribute');
$attribute_query->setQuery(new Elastica\Query\Match('attribute.id', $id));
$value_filter = new Elastica\Filter\Query();
$value_filter->setQuery($attribute_query);
$max = new Elastica\Aggregation\Max('max_number');
$max->setField('_number');
$min = new Elastica\Aggregation\Min('min_number');
$min->setField('_number');
$aggs = new Elastica\Aggregation\Filter('numeric_range', $value_filter);
$aggs->addAggregation($max)->addAggregation($min);
$query = new Elastica\Query();
$query->setSize(0);
$query->addAggregation($aggs);
/* @var $elastica_client Elastica\Client */
$elastica_client = $this->getServiceLocator()->get('elastica-client');
$result['query'] = $query->toArray();
try {
/* @var $response \Elastica\Response */
$response = $elastica_client->request('reports/value/_search', Request::GET, $query->toArray());
$data = $response->getData();
$limits = ['min' => 0, 'max' => 0];
if (isset($data['aggregations']['numeric_range']) && $data['aggregations']['numeric_range']['doc_count'] > 0) {
$limits['min'] = $data['aggregations']['numeric_range']['min_number']['value'];
$limits['max'] = $data['aggregations']['numeric_range']['max_number']['value'];
}
$result['data'] = $limits;
$result['outcome'] = 1;
} catch (\Exception $e) {
$result['error'] = $e->getMessage();
}
break;
}
}
return new JsonModel($result);
}
/**
* @group functional
*/
public function testGlobalSearchTypeSearchWithKeys()
{
$type = $this->_createType();
$index = $type->getIndex();
$client = $index->getClient();
$multiSearch = new MultiSearch($client);
$search1 = new Search($client);
$search1->addIndex($index)->addType($type);
$query1 = new Query();
$termQuery1 = new Term();
$termQuery1->setTerm('username', 'farrelley');
$query1->setQuery($termQuery1);
$query1->setSize(2);
$search1->setQuery($query1);
$multiSearch->addSearch($search1);
$this->assertCount(1, $multiSearch->getSearches());
$search2 = new Search($client);
$search2->addIndex($index)->addType($type);
$query2 = new Query();
$termQuery2 = new Term();
$termQuery2->setTerm('username', 'bunny');
$query2->setQuery($termQuery2);
$query2->setSize(3);
$search2->setQuery($query2);
$multiSearch->addSearch($search2);
$multiSearch->setSearchType(Search::OPTION_SEARCH_TYPE_COUNT);
$multiResultSet = $multiSearch->search();
$this->assertInstanceOf('Elastica\\Multi\\ResultSet', $multiResultSet);
$this->assertCount(2, $multiResultSet);
$this->assertInstanceOf('Elastica\\Response', $multiResultSet->getResponse());
$resultSets = $multiResultSet->getResultSets();
$this->assertInternalType('array', $resultSets);
$this->assertArrayHasKey(0, $resultSets);
$this->assertInstanceOf('Elastica\\ResultSet', $resultSets[0]);
$this->assertCount(0, $resultSets[0]);
$this->assertSame($query1, $resultSets[0]->getQuery());
$this->assertEquals(3, $resultSets[0]->getTotalHits());
$this->assertArrayHasKey(1, $resultSets);
$this->assertInstanceOf('Elastica\\ResultSet', $resultSets[1]);
$this->assertCount(0, $resultSets[1]);
$this->assertSame($query2, $resultSets[1]->getQuery());
$this->assertEquals(6, $resultSets[1]->getTotalHits());
$search1->setOption(Search::OPTION_SEARCH_TYPE, Search::OPTION_SEARCH_TYPE_QUERY_AND_FETCH);
$multiResultSet = $multiSearch->search();
$this->assertInstanceOf('Elastica\\Multi\\ResultSet', $multiResultSet);
$this->assertCount(2, $multiResultSet);
$this->assertInstanceOf('Elastica\\Response', $multiResultSet->getResponse());
$resultSets = $multiResultSet->getResultSets();
$this->assertInternalType('array', $resultSets);
$this->assertArrayHasKey(0, $resultSets);
$this->assertInstanceOf('Elastica\\ResultSet', $resultSets[0]);
$this->assertCount(2, $resultSets[0]);
$this->assertSame($query1, $resultSets[0]->getQuery());
$this->assertEquals(3, $resultSets[0]->getTotalHits());
$this->assertArrayHasKey(1, $resultSets);
$this->assertInstanceOf('Elastica\\ResultSet', $resultSets[1]);
$this->assertCount(0, $resultSets[1]);
$this->assertSame($query2, $resultSets[1]->getQuery());
$this->assertEquals(6, $resultSets[1]->getTotalHits());
}
/**
* {@inheritdoc}
*/
public function generateQueryBy(array $criteria, array $orderBy = null, $limit = null, $offset = null)
{
$query = new Query();
if (empty($criteria)) {
$query->setQuery(new MatchAll());
} else {
$query->setQuery(new Filtered(null, $this->generateFilterBy($criteria)));
}
if ($orderBy) {
$query->setSort($orderBy);
}
if ($limit) {
$query->setSize($limit);
}
if ($offset) {
$query->setFrom($offset);
}
return $query;
}
/**
* Powers full-text-like searches including prefix search.
*
* @param string $type
* @param string $for
* @return Status(mixed) results from the query transformed by the resultsType
*/
private function search($type, $for)
{
if ($this->nonTextQueries) {
$bool = new \Elastica\Query\Bool();
if ($this->query !== null) {
$bool->addMust($this->query);
}
foreach ($this->nonTextQueries as $nonTextQuery) {
$bool->addMust($nonTextQuery);
}
$this->query = $bool;
}
if ($this->resultsType === null) {
$this->resultsType = new FullTextResultsType(FullTextResultsType::HIGHLIGHT_ALL);
}
// Default null queries now so the rest of the method can assume it is not null.
if ($this->query === null) {
$this->query = new \Elastica\Query\MatchAll();
}
$query = new Elastica\Query();
$query->setParam('_source', $this->resultsType->getSourceFiltering());
$query->setParam('fields', $this->resultsType->getFields());
$extraIndexes = array();
$indexType = $this->pickIndexTypeFromNamespaces();
if ($this->namespaces) {
$extraIndexes = $this->getAndFilterExtraIndexes();
if ($this->needNsFilter($extraIndexes, $indexType)) {
$this->filters[] = new \Elastica\Filter\Terms('namespace', $this->namespaces);
}
}
// Wrap $this->query in a filtered query if there are any filters
$unifiedFilter = Filters::unify($this->filters, $this->notFilters);
if ($unifiedFilter !== null) {
$this->query = new \Elastica\Query\Filtered($this->query, $unifiedFilter);
}
// Call installBoosts right after we're done munging the query to include filters
// so any rescores installBoosts adds to the query are done against filtered results.
$this->installBoosts();
$query->setQuery($this->query);
$highlight = $this->resultsType->getHighlightingConfiguration($this->highlightSource);
if ($highlight) {
// Fuzzy queries work _terribly_ with the plain highlighter so just drop any field that is forcing
// the plain highlighter all together. Do this here because this works so badly that no
// ResultsType should be able to use the plain highlighter for these queries.
if ($this->fuzzyQuery) {
$highlight['fields'] = array_filter($highlight['fields'], function ($field) {
return $field['type'] !== 'plain';
});
}
if (!empty($this->nonTextHighlightQueries)) {
// We have some phrase_prefix queries, so let's include them in the
// generated highlight_query.
$bool = new \Elastica\Query\Bool();
if ($this->highlightQuery) {
$bool->addShould($this->highlightQuery);
}
foreach ($this->nonTextHighlightQueries as $nonTextHighlightQuery) {
$bool->addShould($nonTextHighlightQuery);
}
$this->highlightQuery = $bool;
}
if ($this->highlightQuery) {
$highlight['highlight_query'] = $this->highlightQuery->toArray();
}
$query->setHighlight($highlight);
}
if ($this->suggest) {
$query->setParam('suggest', $this->suggest);
$query->addParam('stats', 'suggest');
}
if ($this->offset) {
$query->setFrom($this->offset);
}
if ($this->limit) {
$query->setSize($this->limit);
}
if ($this->sort != 'relevance') {
// Clear rescores if we aren't using relevance as the search sort because they aren't used.
$this->rescore = array();
}
if ($this->rescore) {
// rescore_query has to be in array form before we send it to Elasticsearch but it is way easier to work
// with if we leave it in query for until now
$modifiedRescore = array();
foreach ($this->rescore as $rescore) {
$rescore['query']['rescore_query'] = $rescore['query']['rescore_query']->toArray();
$modifiedRescore[] = $rescore;
}
$query->setParam('rescore', $modifiedRescore);
}
$query->addParam('stats', $type);
switch ($this->sort) {
case 'relevance':
break;
// The default
// The default
case 'title_asc':
$query->setSort(array('title.keyword' => 'asc'));
break;
case 'title_desc':
$query->setSort(array('title.keyword' => 'desc'));
break;
case 'incoming_links_asc':
$query->setSort(array('incoming_links' => array('order' => 'asc', 'missing' => '_first')));
break;
case 'incoming_links_desc':
$query->setSort(array('incoming_links' => array('order' => 'desc', 'missing' => '_last')));
break;
default:
LoggerFactory::getInstance('CirrusSearch')->warning("Invalid sort type: {sort}", array('sort' => $this->sort));
}
$queryOptions = array();
if ($this->config->get('CirrusSearchMoreAccurateScoringMode')) {
$queryOptions['search_type'] = 'dfs_query_then_fetch';
}
switch ($type) {
case 'regex':
$poolCounterType = 'CirrusSearch-Regex';
$queryOptions['timeout'] = $this->config->getElement('CirrusSearchSearchShardTimeout', 'regex');
break;
case 'prefix':
$poolCounterType = 'CirrusSearch-Prefix';
$queryOptions['timeout'] = $this->config->getElement('CirrusSearchSearchShardTimeout', 'default');
break;
default:
$poolCounterType = 'CirrusSearch-Search';
$queryOptions['timeout'] = $this->config->getElement('CirrusSearchSearchShardTimeout', 'default');
}
$this->connection->setTimeout($queryOptions['timeout']);
// Setup the search
$pageType = $this->connection->getPageType($this->indexBaseName, $indexType);
$search = $pageType->createSearch($query, $queryOptions);
foreach ($extraIndexes as $i) {
$search->addIndex($i);
}
$description = "{queryType} search for '{query}'";
$logContext = array('queryType' => $type, 'query' => $for);
if ($this->returnQuery) {
return Status::newGood(array('description' => $this->formatDescription($description, $logContext), 'path' => $search->getPath(), 'params' => $search->getOptions(), 'query' => $query->toArray(), 'options' => $queryOptions));
}
if ($this->returnExplain && $this->returnResult) {
$query->setExplain(true);
}
// Perform the search
$searcher = $this;
$user = $this->user;
$result = Util::doPoolCounterWork($poolCounterType, $this->user, function () use($searcher, $search, $description, $logContext) {
try {
$searcher->start($description, $logContext);
return $searcher->success($search->search());
} catch (\Elastica\Exception\ExceptionInterface $e) {
return $searcher->failure($e);
}
}, function ($error, $key, $userName) use($type, $description, $user, $logContext) {
$forUserName = $userName ? "for {userName} " : '';
LoggerFactory::getInstance('CirrusSearch')->warning("Pool error {$forUserName}on key {key} during {$description}: {error}", $logContext + array('userName' => $userName, 'key' => 'key', 'error' => $error));
if ($error === 'pool-queuefull') {
if (strpos($key, 'nowait:CirrusSearch:_per_user') === 0) {
$loggedIn = $user->isLoggedIn() ? 'logged-in' : 'anonymous';
return Status::newFatal("cirrussearch-too-busy-for-you-{$loggedIn}-error");
}
if ($type === 'regex') {
return Status::newFatal('cirrussearch-regex-too-busy-error');
}
return Status::newFatal('cirrussearch-too-busy-error');
}
return Status::newFatal('cirrussearch-backend-error');
});
if ($result->isOK()) {
$responseData = $result->getValue()->getResponse()->getData();
if ($this->returnResult) {
return Status::newGood(array('description' => $this->formatDescription($description, $logContext), 'path' => $search->getPath(), 'result' => $responseData));
}
$result->setResult(true, $this->resultsType->transformElasticsearchResult($this->suggestPrefixes, $this->suggestSuffixes, $result->getValue(), $this->searchContext->isSearchContainedSyntax()));
if (isset($responseData['timed_out']) && $responseData['timed_out']) {
LoggerFactory::getInstance('CirrusSearch')->warning("{$description} timed out and only returned partial results!", $logContext);
if ($result->getValue()->numRows() === 0) {
return Status::newFatal('cirrussearch-backend-error');
} else {
$result->warning('cirrussearch-timed-out');
}
}
}
return $result;
}
public function findOneBy(ClassMetadata $class, $field, $value)
{
$filter = new Term(array($field => $value));
$query = new Query(new Filtered(null, $filter));
$query->setVersion(true);
$query->setSize(1);
$results = $this->search($query, array($class));
if (!$results->count()) {
throw new NoResultException();
}
return $results[0];
}
/**
* @Extra\Route(
* "/api/twitter/aggregate/{keywords}/{since}/{until}",
* name="weaving_the_web_dashboard_people_talking_about",
* requirements={"keywords": "[-,%\+a-zA-Z0-9]+"}
* )
* @Extra\Method({"GET"})
*
* @Extra\Cache(expires="+1 week", public="true")
*
* @param $keywords
* @param \DateTime $since
* @param \DateTime $until
* @return \Symfony\Component\HttpFoundation\Response
*/
public function aggregateFilteredTermsAction($keywords, \DateTime $since, \DateTime $until)
{
$termsAggregationName = 'screen_name_aggregations';
$filteredAggregationName = 'screen_name_aggregated_in_range';
$timeSeries = [];
$screenNamesAggregations = [];
$keywords = explode(',', $keywords);
$keywordIndex = 0;
if (count($keywords) > 5) {
$keywords = array_slice($keywords, 0, 5);
}
$fingerprint = sha1(serialize([$keywords, $since, $until]));
$aggregationsPath = sprintf(__DIR__ . '/../Resources/json/aggregations/%s.json', $fingerprint);
if (file_exists($aggregationsPath)) {
$content = unserialize(base64_decode(file_get_contents($aggregationsPath)));
return new JsonResponse($content);
}
$lastYear = $until->format('Y');
foreach ($keywords as $keyword) {
$match = new Query\Match();
$match->setField('text', $keyword);
$query = new Query($match);
$aggregation = new Aggregation();
$timeSeries[$keywordIndex] = [];
foreach (range($since->format('Y'), $lastYear) as $year) {
if ($year === $lastYear) {
$lastMonth = min(12, $until->format('m'));
} else {
$lastMonth = 12;
}
foreach (range(1, $lastMonth) as $month) {
if ($year === $lastYear && $month == $lastMonth) {
$days = $until->format('d');
} else {
$days = cal_days_in_month(CAL_GREGORIAN, $month, $year);
}
foreach (range(1, $days) as $day) {
$termsAggregation = $aggregation->terms($termsAggregationName);
$termsAggregation->setField('screenName');
$termsAggregation->setSize(30);
$yearMonthDay = $year . '-' . str_pad($month, 2, '0', STR_PAD_LEFT) . '-' . str_pad($day, 2, '0', STR_PAD_LEFT);
$startDate = new \DateTime($yearMonthDay . ' 0:00');
$endDate = new \DateTime($yearMonthDay . ' 23:59');
$range = new Range('createdAt', ['gte' => $startDate->format('c'), 'lte' => $endDate->format('c')]);
$rangeFilterAggregation = $aggregation->filter($filteredAggregationName, $range);
$rangeFilterAggregation->addAggregation($termsAggregation);
$query->addAggregation($rangeFilterAggregation);
$query->setSize(100);
$searchIndex = $this->container->getParameter('twitter_search_index');
/** @var \FOS\ElasticaBundle\Elastica\Index $index */
$index = $this->get('fos_elastica.index.' . $searchIndex);
$userStatusType = $index->getType('user_status');
$aggregations = $userStatusType->search($query)->getAggregations();
foreach ($aggregations[$filteredAggregationName][$termsAggregationName]['buckets'] as $bucket) {
$screenNamesAggregations[$yearMonthDay][$keywordIndex][$bucket['key']] = $bucket['doc_count'];
}
$timeSeries[$keywordIndex][] = ['date' => $yearMonthDay, 'mentions' => $aggregations[$filteredAggregationName]['doc_count']];
}
}
}
$keywordIndex++;
}
$contents = ['time_series' => $timeSeries, 'screen_name_aggregations' => $screenNamesAggregations];
file_put_contents($aggregationsPath, base64_encode(serialize($contents)));
return new JsonResponse($contents);
}
/**
*
* @param array $location
*
* @return Elastica\Query $localityQuery
*/
public static function getLocalityQuery($location)
{
$query = new Agent\Elastica\Query\BoolQuery();
$method = 'addMust';
if (!isset($location['state']) && !isset($location['zip']) && !isset($location['locality'])) {
foreach (['phone', 'ipaddress'] as $field) {
if (isset($location[$field]) && empty($location['state'])) {
switch ($field) {
case 'ipaddress':
$geo = self::$geo;
$loc = $geo->getRecord($location['ipaddress']);
if ($loc instanceof Record) {
$state = $loc->getRegion();
if ($state) {
$location['state'] = $state;
}
}
break;
case 'phone':
$phone = Helper::parse_phonenumber($location['phone'], 'array');
if ($phone) {
$state = Helper::area_code_to_state($phone[0]);
if ($state) {
$location['state'] = $state;
}
}
break;
}
}
}
}
foreach ($location as $field => $value) {
switch ($field) {
case 'locality':
if (!isset($location['zip'])) {
$fields = ['latitude', 'longitude'];
$values = is_array($value) ? $value : explode(",", $value);
$latlon = count($values) == 2 ? array_combine($fields, $values) : false;
if ($latlon) {
$path = "location";
$nested = new Elastica\Query\Nested();
$nested->setPath($path);
$bool = new Elastica\Query\BoolQuery();
foreach ($latlon as $dim => $coord) {
$bool->addMust(new Elastica\Query\Match("{$path}.{$dim}", $coord));
}
$nested->setQuery($bool);
$query->addMust($nested);
}
}
break;
case 'city':
if (!isset($location['locality'])) {
$query->addShould(new Elastica\Query\Match($field, $value));
}
break;
case 'state':
if (!isset($location['locality'])) {
$fields = ['state.abbrev', 'state.full'];
$values = is_array($value) ? $value : [$value];
foreach ($values as $state) {
$querystring = new Elastica\Query\QueryString($state);
$querystring->setFields($fields);
$nested = new Elastica\Query\Nested();
$nested->setQuery($querystring);
$nested->setPath($field);
if (count($values) > 1) {
$query->addShould($nested);
} else {
$query->addMust($nested);
}
}
}
break;
case 'zip':
$query->{$method}(new Elastica\Query\Match($field, $value));
break;
}
}
$localityQuery = new Elastica\Query($query);
$localityQuery->setSize(1);
return $localityQuery;
}
public function search()
{
//$finder = $this->container->get('fos_elastica.finder.search.articles');
$bool = new Query\Bool();
$multiMatch = new Query\MultiMatch();
$multiMatch->setFields(['subjects', 'title', 'keywords', 'subtitle', 'citations.raw', 'journal.title', 'journal.subtitle']);
$multiMatch->setType('phrase_prefix');
$multiMatch->setQuery($this->getParam('term'));
$bool->addMust($multiMatch);
if ($this->filter) {
foreach ($this->filter as $key => $filter) {
$filterObj = new Query\Match();
$this->applyFilter($filterObj, $key, $filter);
$bool->addMust($filterObj);
}
}
$missing = new Filter\Missing("issue");
$not = new Filter\BoolNot($missing);
$notQ = new Query\Filtered();
$notQ->setFilter($not);
$bool->addMust($notQ);
$query = new Query();
$query->setQuery($bool);
$query->setFrom($this->getPage() * $this->getLimit());
$query->setSize($this->getLimit());
$aggregation = new Terms('journals');
$aggregation->setField('journal.id');
$aggregation->setOrder('_count', 'desc');
$qb = $this->em->createQueryBuilder();
$qb->select('count(r.id)')->from('OjsJournalBundle:Journal', 'r')->where($qb->expr()->eq('r.status', 3));
$aggregation->setSize($qb->getQuery()->getSingleScalarResult());
$query->addAggregation($aggregation);
$aggregation = new Terms('authors');
$aggregation->setField('articleAuthors.author.id');
$aggregation->setOrder('_count', 'desc');
$qb = $this->em->createQueryBuilder();
$qb->select('count(r.id)')->from('OjsJournalBundle:Author', 'r');
$aggregation->setSize($qb->getQuery()->getSingleScalarResult());
$query->addAggregation($aggregation);
$elasticaAdapter = new ElasticaAdapter($this->index, $query);
$pagerFanta = new Pagerfanta($elasticaAdapter);
$pagerFanta->setMaxPerPage($this->getLimit());
$pagerFanta->setCurrentPage($this->getPage());
/** @var ResultSet $search */
$search = $pagerFanta->getCurrentPageResults();
$result = $search->getResults();
//$search->getResults();
$this->pager = $pagerFanta;
$transformer = new ElasticaToModelTransformer($this->registry, 'OjsJournalBundle:Article');
$transformer->setPropertyAccessor($this->propertyAccessor);
$this->result = $transformer->transform($result);
$this->setCount($pagerFanta->getNbResults());
$this->addAggregation('journal', $this->transform($search->getAggregation('journals')['buckets'], 'OjsJournalBundle:Journal'));
$this->addAggregation('author', $this->transform($search->getAggregation('authors')['buckets'], 'OjsJournalBundle:Author'));
return $this;
}
/**
* @inheritDoc
*/
public function paginate()
{
$this->query->setSize($this->limit);
$this->query->setFrom(($this->page - 1) * $this->limit);
return $this->query;
}
/**
* @param string $term
* @param string $value
* @param int $size
*
* @return \Elastica\ResultSet
*/
public function termQuery($term, $value, $size = 30)
{
$_agg = new Aggregation();
$_facet = $_agg->date_histogram('occurred_on', '@timestamp', ElkIntervals::DAY);
// $_facet = new DateHistogram('occurred_on');
// $_facet->setField('@timestamp');
// $_facet->setInterval('day');
$_query = new Query();
$_query->setSize($size);
$_query->setSort(['@timestamp']);
// Filter for term
$_filter = new Prefix($term, $value);
$_and = new Bool();
$_and->addMust($_filter);
$_query->setPostFilter($_and);
$_query->addAggregation($_facet);
$_results = $this->_doSearch($_query);
return $_results;
}
/**
*
* @param Report $report
* @param number $limit
* @param string $sort
* @param string $order
* @param boolean $lazy
* @param boolean $active
* @param boolean $silent
*
* @return ArrayCollection $data
*/
public function generateResults(Report $report, $limit = 0, $sort = '_score', $order = 'desc', $lazy = true, $active = true, $silent = false)
{
$data = new ArrayCollection();
$request = $this->getServiceLocator()->get('Request');
$silent = $request->getQuery('debug') ? false : $silent;
if ($report instanceof Report) {
$agent = $report->getAgent();
$lead_query = new \Agent\Elastica\Query\BoolQuery();
if ($active) {
$lead_query->addMust(new Elastica\Query\Match('active', 1));
}
$client = $this->getElasticaClient();
if ($agent && $client) {
$filters = $agent->getFilter();
if ($filters) {
$lead_query = $this->applyFilters($lead_query, $filters);
}
$criteria = $agent->getCriteria(false);
if ($criteria) {
try {
$query = new Elastica\Query();
/* @var $criterion \Agent\Entity\AgentCriterion */
foreach ($criteria as $i => $criterion) {
$lead_query = $this->buildQuery($lead_query, $criterion);
}
$query->setQuery($lead_query);
$size = $limit ? $limit : 1000;
$query->setParam('track_scores', true);
$query->addSort([$sort => ['order' => $order]]);
$query->setSize($size);
$results = $this->runQuery($query, $lazy, $silent);
$total = isset($results['total']) ? $results['total'] : false;
if ($total && $results['results']) {
foreach ($results['results'] as $result) {
$data->add($result);
}
}
if ($total > $size) {
$limit = $limit ?: $total;
for ($page = 1; $page < ceil($limit / $size); $page++) {
$query->setFrom($page * $size);
$results = $this->runQuery($query, $silent);
if ($results['results']) {
foreach ($results as $result) {
if (is_array($result)) {
foreach ($result as $r) {
if ($r instanceof Result) {
$data->add($r);
}
}
} elseif ($result instanceof Result) {
$data->add($result);
}
}
}
}
}
} catch (\Exception $e) {
if (!$silent) {
$this->getFlashMessenger()->addErrorMessage($e->getMessage());
}
}
}
}
}
return $data;
}
/**
* Perform a search based on the specified filter
*
* @param $typeTerms array
* @param null $user
* @param null $sortColumn
* @param string $sortDirection
* @param int $limit
* @return array
*/
public function filteredSearch($typeTerms, $user = null, $sortColumn = null, $sortDirection = 'desc', $limit = 50)
{
$query = new Query();
$query->setFrom(0);
$query->setSize($limit);
$boolQuery = new BoolQuery();
// Handle 'must' filters
if (isset($typeTerms['must'])) {
foreach ($typeTerms['must'] as $type => $term) {
$termQueryType = new TermQuery([$type => $term]);
$boolQuery->addMust($termQueryType);
}
}
// Handle 'should' filters
if (isset($typeTerms['should'])) {
foreach ($typeTerms['should'] as $type => $term) {
$termQueryType = new TermQuery([$type => $term]);
$boolQuery->addShould($termQueryType);
}
}
$query->setQuery($boolQuery);
if ($user) {
$userId = $user->id;
$termQueryUser = new TermQuery(['user_id' => $userId]);
$boolQuery->addMust($termQueryUser);
}
if ($sortColumn) {
$query->addSort([$sortColumn => ['order' => $sortDirection]]);
}
$resultSet = $this->getTypedIndex()->search($query);
return $this->getModels($resultSet);
}
/**
* @param $searchClosed
* @return \Elastica\ResultSet
* @throws Exception
*/
public function doSearch($searchClosed)
{
$this->connection = new ElasticSearchConnection();
$this->connection->init();
$this->whereClause = new Query\QueryString();
$this->whereClause->setQuery($this->searchTerms);
$this->utility = new Util();
if (isset($_GET['page'])) {
$this->currentPage = intval($_GET['page']);
}
$this->fieldMapping = $this->configurationManager->getConfiguration(ConfigurationManager::CONFIGURATION_TYPE_SETTINGS, 'WMDB.Forger.SearchTermMapping');
$elasticaQuery = new Query();
if ($searchClosed === 'true') {
$elasticaQuery->setQuery($this->whereClause);
} else {
$boolSearch = new Query\Bool();
$boolSearch->addMust($this->whereClause);
$boolSearch->addMustNot(['term' => ['status.name' => 'Closed']]);
$boolSearch->addMustNot(['term' => ['status.name' => 'Rejected']]);
$boolSearch->addMustNot(['term' => ['status.name' => 'Resolved']]);
$elasticaQuery->setQuery($boolSearch);
}
$elasticaQuery->setSize($this->perPage);
$elasticaQuery->setFrom($this->currentPage * $this->perPage - $this->perPage);
$usedFilters = $this->addFilters();
if ($usedFilters !== false) {
$elasticaQuery->setPostFilter($usedFilters);
}
$this->addAggregations($elasticaQuery);
$elasticaResultSet = $this->connection->getIndex()->search($elasticaQuery);
$results = $elasticaResultSet->getResults();
$maxScore = $elasticaResultSet->getMaxScore();
$aggs = $elasticaResultSet->getAggregations();
$this->totalHits = $elasticaResultSet->getTotalHits();
$out = array('pagesToLinkTo' => $this->getPages(), 'currentPage' => $this->currentPage, 'prev' => $this->currentPage - 1, 'next' => $this->currentPage < ceil($this->totalHits / $this->perPage) ? $this->currentPage + 1 : 0, 'totalResults' => $this->totalHits, 'startingAtItem' => $this->currentPage * $this->perPage - ($this->perPage - 1), 'endingAtItem' => $this->currentPage * $this->perPage, 'results' => $results, 'maxScore' => $maxScore, 'aggs' => $aggs);
if (intval($this->totalHits) <= intval($out['endingAtItem'])) {
$out['endingAtItem'] = intval($this->totalHits);
}
return $out;
}
public function searchAction()
{
//WIP, todo: export to repository and use it in Search Playlists
$elasticaClient = new Client();
$playListIndex = $elasticaClient->getIndex('playlist');
$trackType = $playListIndex->getType('track');
$search = new Search($elasticaClient);
$search->addIndex($playListIndex)->addType($trackType);
$query = new Query();
$query->setSize(5)->setSort(['name' => 'asc'])->setFields(['name', 'ids', 'id', 'composer'])->setExplain(true)->setVersion(true)->setHighlight(['fields' => 'composer']);
$query->setQuery(new MatchAll());
// $query->addAggregation(new \Elastica\Aggregation\Range('name'));
// $term = new \Elastica\Suggest\Term('name', 'field');
// $term->setText('aaaaa');
// $query->setSuggest(new \Elastica\Suggest($term));
// $query->setFacets([new \Elastica\Facet\Range('name')]);
$search->setQuery($query);
$resultSet = $search->search();
$numberOfEntries = $search->count();
$results = $resultSet->getResults();
$totalResults = $resultSet->getTotalHits();
return $this->render('PlayWithElasticSearchBundle:Elastica:search.html.twig', ['query' => $query, 'numberOfEntries' => $numberOfEntries, 'resultSet' => $resultSet, 'results' => $results, 'totalResults' => $totalResults]);
}
