public function searchAction(Request $request) { $queryString = $request->get('query'); // count total documents indexed $numDocs = $this->get('zendsearch')->numDocs(); // parse query string and return a Query object. // $query = Search\QueryParser::parse($queryString, 'UTF-8'); $queryTokens = Analyzer::getDefault()->tokenize($queryString, 'UTF-8'); $query = new Search\Query\Boolean(); foreach ($queryTokens as $token) { $query->addSubquery(new Search\Query\Fuzzy(new Index\Term($token->getTermText()), 0.5), null); } // process query $results = $this->get('zendsearch')->find($query); // sort results by score (MultiSearch does not sort the results between the differents indices) usort($results, create_function('$a, $b', 'return $a->score < $b->score;')); // // paginate results // $results = new \Zend\Paginator\Paginator(new \Zend\Paginator\Adapter\ArrayAdapter($results)); // $results->setCurrentPageNumber($page); // $results->setItemCountPerPage($rpp); // // fetch results entities // $dataResults = array(); // foreach ($results as $hit) { // $document = $hit->getDocument(); // $repository = $this->get('orm.em')->getRepository( $document->getFieldValue('entityClass') ); // $dataResults[] = $repository->find( $document->getFieldValue('id') ); // } // $results = $dataResults; return $this->get('twig')->render('admin/search.html.twig', array('query' => $queryString, 'numDocs' => $numDocs, 'results' => $results)); }
public function prepareQuery($expressionOrigin, $conditions) { setlocale(LC_ALL, "cs_CZ.UTF-8"); $expressionOrigin = strtr($expressionOrigin, array(',' => '', ';' => '', "'" => '', '"' => '', '-' => '', '_' => '', '/' => '', '\\' => '', '+' => '', '=' => '', '?' => '', '.' => '', '!' => '')); $expressionTranslit = str_replace("'", '', iconv('UTF-8', 'ASCII//TRANSLIT//IGNORE', $expressionOrigin)); $expressions = array($expressionOrigin); if ($expressionOrigin != $expressionTranslit) { $expressions[] = $expressionTranslit; } $queryWords = array(); $query = new Boolean(); foreach ($expressions as $expression) { // more words in expression if (count($expressionWords = explode(' ', $expression)) > 1) { // whole expression $query->addSubquery(QueryParser::parse('"' . $expression . '"', 'utf-8')); // expression words foreach ($expressionWords as $expressionWord) { if (mb_strlen($expressionWord, 'utf-8') > 2 && !in_array($expressionWord, $queryWords)) { $queryWords[] = $expressionWord; $query->addSubquery(QueryParser::parse($expressionWord . '*', 'utf-8')); } } } else { $query->addSubquery(QueryParser::parse($expression . '*', 'utf-8')); } } // specificke podminky do query if (is_array($conditions) && count($conditions)) { foreach ($conditions as $condition) { // TODO - jak v Lucene najit polozky obsahujici url? Wildcard pouzit nejde... if (mb_strpos($condition, 'url:', null, 'utf-8') === 0) { $uri = trim(substr($condition, 4), '"'); if (strpos($uri, '://') !== false) { $uri = substr($uri, strpos($uri, '://') + 3); $uri = substr($uri, strpos($uri, '/')); } $query .= ' AND ' . Page::URIS_KEY . ':"' . $uri . '"'; } else { $query .= ' AND ' . $condition; } } } if ($this->kernel->getEnvironment() == 'dev') { $session = new Session(); $session->set(self::QUERY_HANDLER, $query); } return $query; }
public function testSearchRawQuery() { $query = Search::rawQuery('description:big'); $this->assertEquals(2, $query->count()); $query = Search::rawQuery(function () { return 'description:big'; }); $this->assertEquals(2, $query->count()); $query = Search::rawQuery(function () { $query = new Boolean(); $query->addSubquery(QueryParser::parse('description:big OR name:monitor')); return $query; }); $this->assertEquals(3, $query->count()); }
/** * Add a search/where clause to the given query based on the given condition. * Return the given $query instance when finished. * * @param \ZendSearch\Lucene\Search\Query\Boolean $query * @param array $condition - field : name of the field * - value : value to match * - required : must match * - prohibited : must not match * - phrase : match as a phrase * - filter : filter results on value * - fuzzy : fuzziness value (0 - 1) * * @return \ZendSearch\Lucene\Search\Query\Boolean */ public function addConditionToQuery($query, array $condition) { if (array_get($condition, 'lat')) { return $query; } $value = trim($this->escape(array_get($condition, 'value'))); if (array_get($condition, 'phrase') || array_get($condition, 'filter')) { $value = '"' . $value . '"'; } if (isset($condition['fuzzy']) && false !== $condition['fuzzy']) { $fuzziness = ''; if (is_numeric($condition['fuzzy']) && $condition['fuzzy'] >= 0 && $condition['fuzzy'] <= 1) { $fuzziness = $condition['fuzzy']; } $words = array(); foreach (explode(' ', $value) as $word) { $words[] = $word . '~' . $fuzziness; } $value = implode(' ', $words); } $sign = null; if (!empty($condition['required'])) { $sign = true; } else { if (!empty($condition['prohibited'])) { $sign = false; } } $field = array_get($condition, 'field'); if (empty($field) || '*' === $field) { $field = null; } if (is_array($field)) { $values = array(); foreach ($field as $f) { $values[] = trim($f) . ':(' . $value . ')'; } $value = implode(' OR ', $values); } else { if ($field) { $value = trim(array_get($condition, 'field')) . ':(' . $value . ')'; } } $query->addSubquery(\ZendSearch\Lucene\Search\QueryParser::parse($value), $sign); return $query; }
function testUpdate() { // preparation $app = new Application(); $container = $app->getContainer(); // get an index /** @var Index $index */ $index = $container->query('Index'); // add a document $doc = new Document(); $doc->addField(Document\Field::Keyword('fileId', '1')); $doc->addField(Document\Field::Text('path', '/somewhere/deep/down/the/rabbit/hole', 'UTF-8')); $doc->addField(Document\Field::Text('users', 'alice', 'UTF-8')); $index->index->addDocument($doc); $index->commit(); // search for it $idTerm = new Term('1', 'fileId'); $idQuery = new Query\Term($idTerm); $query = new Query\Boolean(); $query->addSubquery($idQuery); /** @var QueryHit $hit */ $hits = $index->find($query); // get the document from the query hit $foundDoc = $hits[0]->getDocument(); $this->assertEquals('alice', $foundDoc->getFieldValue('users')); // delete the document from the index //$index->index->delete($hit); // change the 'users' key of the document $foundDoc->addField(Document\Field::Text('users', 'bob', 'UTF-8')); $this->assertEquals('bob', $foundDoc->getFieldValue('users')); // add the document back to the index $index->updateFile($foundDoc, '1'); $idTerm2 = new Term('1', 'fileId'); $idQuery2 = new Query\Term($idTerm2); $query2 = new Query\Boolean(); $query2->addSubquery($idQuery2); /** @var QueryHit $hit */ $hits2 = $index->find($query2); // get the document from the query hit $foundDoc2 = $hits2[0]->getDocument(); $this->assertEquals('bob', $foundDoc2->getFieldValue('users')); }
/** * Add subquery to boolean query. * * @param QueryBoolean $query * @param array $options * @return QueryBoolean * @throws \RuntimeException */ protected function addSubquery($query, array $options) { list($value, $sign) = $this->queryBuilder->build($options); $query->addSubquery($this->queryBuilder->parse($value), $sign); return $query; }
/** * Re-write query into primitive queries in the context of specified index * * @param \ZendSearch\Lucene\SearchIndexInterface $index * @return \ZendSearch\Lucene\Search\Query\AbstractQuery */ public function rewrite(Lucene\SearchIndexInterface $index) { if (count($this->_terms) == 0) { return new EmptyResult(); } // Check, that all fields are qualified $allQualified = true; foreach ($this->_terms as $term) { if ($term->field === null) { $allQualified = false; break; } } if ($allQualified) { return $this; } else { /** transform multiterm query to boolean and apply rewrite() method to subqueries. */ $query = new Boolean(); $query->setBoost($this->getBoost()); foreach ($this->_terms as $termId => $term) { $subquery = new Term($term); $query->addSubquery($subquery->rewrite($index), $this->_signs === null ? true : $this->_signs[$termId]); } return $query; } }
/** * Re-write query into primitive queries in the context of specified index * * @param \ZendSearch\Lucene\SearchIndexInterface $index * @throws \ZendSearch\Lucene\Exception\OutOfBoundsException * @return \ZendSearch\Lucene\Search\Query\AbstractQuery */ public function rewrite(Lucene\SearchIndexInterface $index) { $this->_matches = array(); $this->_scores = array(); $this->_termKeys = array(); if ($this->_term->field === null) { // Search through all fields $fields = $index->getFieldNames(true); } else { $fields = array($this->_term->field); } $prefix = Index\Term::getPrefix($this->_term->text, $this->_prefixLength); $prefixByteLength = strlen($prefix); $prefixUtf8Length = Index\Term::getLength($prefix); $termLength = Index\Term::getLength($this->_term->text); $termRest = substr($this->_term->text, $prefixByteLength); // we calculate length of the rest in bytes since levenshtein() is not UTF-8 compatible $termRestLength = strlen($termRest); $scaleFactor = 1 / (1 - $this->_minimumSimilarity); $maxTerms = Lucene\Lucene::getTermsPerQueryLimit(); foreach ($fields as $field) { $index->resetTermsStream(); if ($prefix != '') { $index->skipTo(new Index\Term($prefix, $field)); while ($index->currentTerm() !== null && $index->currentTerm()->field == $field && substr($index->currentTerm()->text, 0, $prefixByteLength) == $prefix) { // Calculate similarity $target = substr($index->currentTerm()->text, $prefixByteLength); $maxDistance = isset($this->_maxDistances[strlen($target)]) ? $this->_maxDistances[strlen($target)] : $this->_calculateMaxDistance($prefixUtf8Length, $termRestLength, strlen($target)); if ($termRestLength == 0) { // we don't have anything to compare. That means if we just add // the letters for current term we get the new word $similarity = $prefixUtf8Length == 0 ? 0 : 1 - strlen($target) / $prefixUtf8Length; } elseif (strlen($target) == 0) { $similarity = $prefixUtf8Length == 0 ? 0 : 1 - $termRestLength / $prefixUtf8Length; } elseif ($maxDistance < abs($termRestLength - strlen($target))) { //just adding the characters of term to target or vice-versa results in too many edits //for example "pre" length is 3 and "prefixes" length is 8. We can see that //given this optimal circumstance, the edit distance cannot be less than 5. //which is 8-3 or more precisesly abs(3-8). //if our maximum edit distance is 4, then we can discard this word //without looking at it. $similarity = 0; } else { $similarity = 1 - levenshtein($termRest, $target) / ($prefixUtf8Length + min($termRestLength, strlen($target))); } if ($similarity > $this->_minimumSimilarity) { $this->_matches[] = $index->currentTerm(); $this->_termKeys[] = $index->currentTerm()->key(); $this->_scores[] = ($similarity - $this->_minimumSimilarity) * $scaleFactor; if ($maxTerms != 0 && count($this->_matches) > $maxTerms) { throw new OutOfBoundsException('Terms per query limit is reached.'); } } $index->nextTerm(); } } else { $index->skipTo(new Index\Term('', $field)); while ($index->currentTerm() !== null && $index->currentTerm()->field == $field) { // Calculate similarity $target = $index->currentTerm()->text; $maxDistance = isset($this->_maxDistances[strlen($target)]) ? $this->_maxDistances[strlen($target)] : $this->_calculateMaxDistance(0, $termRestLength, strlen($target)); if ($maxDistance < abs($termRestLength - strlen($target))) { //just adding the characters of term to target or vice-versa results in too many edits //for example "pre" length is 3 and "prefixes" length is 8. We can see that //given this optimal circumstance, the edit distance cannot be less than 5. //which is 8-3 or more precisesly abs(3-8). //if our maximum edit distance is 4, then we can discard this word //without looking at it. $similarity = 0; } else { $similarity = 1 - levenshtein($termRest, $target) / min($termRestLength, strlen($target)); } if ($similarity > $this->_minimumSimilarity) { $this->_matches[] = $index->currentTerm(); $this->_termKeys[] = $index->currentTerm()->key(); $this->_scores[] = ($similarity - $this->_minimumSimilarity) * $scaleFactor; if ($maxTerms != 0 && count($this->_matches) > $maxTerms) { throw new OutOfBoundsException('Terms per query limit is reached.'); } } $index->nextTerm(); } } $index->closeTermsStream(); } if (count($this->_matches) == 0) { return new EmptyResult(); } elseif (count($this->_matches) == 1) { return new Term(reset($this->_matches)); } else { $rewrittenQuery = new Boolean(); array_multisort($this->_scores, SORT_DESC, SORT_NUMERIC, $this->_termKeys, SORT_ASC, SORT_STRING, $this->_matches); $termCount = 0; foreach ($this->_matches as $id => $matchedTerm) { $subquery = new Term($matchedTerm); $subquery->setBoost($this->_scores[$id]); $rewrittenQuery->addSubquery($subquery); $termCount++; if ($termCount >= self::MAX_CLAUSE_COUNT) { break; } } return $rewrittenQuery; } }
/** * Generate 'boolean style' query from the context * 'term1 and term2 or term3 and (<subquery1>) and not (<subquery2>)' * * @throws \ZendSearch\Lucene\Search\Exception\QueryParserException * @return \ZendSearch\Lucene\Search\Query\AbstractQuery */ private function _booleanExpressionQuery() { /** * We treat each level of an expression as a boolean expression in * a Disjunctive Normal Form * * AND operator has higher precedence than OR * * Thus logical query is a disjunction of one or more conjunctions of * one or more query entries */ $expressionRecognizer = new BooleanExpressionRecognizer(); try { foreach ($this->_entries as $entry) { if ($entry instanceof QueryEntry\AbstractQueryEntry) { $expressionRecognizer->processLiteral($entry); } else { switch ($entry) { case QueryToken::TT_AND_LEXEME: $expressionRecognizer->processOperator(BooleanExpressionRecognizer::IN_AND_OPERATOR); break; case QueryToken::TT_OR_LEXEME: $expressionRecognizer->processOperator(BooleanExpressionRecognizer::IN_OR_OPERATOR); break; case QueryToken::TT_NOT_LEXEME: $expressionRecognizer->processOperator(BooleanExpressionRecognizer::IN_NOT_OPERATOR); break; default: throw new UnexpectedValueException('Boolean expression error. Unknown operator type.'); } } } $conjuctions = $expressionRecognizer->finishExpression(); } catch (ExceptionInterface $e) { // It's query syntax error message and it should be user friendly. So FSM message is omitted throw new QueryParserException('Boolean expression error.', 0, $e); } // Remove 'only negative' conjunctions foreach ($conjuctions as $conjuctionId => $conjuction) { $nonNegativeEntryFound = false; foreach ($conjuction as $conjuctionEntry) { if ($conjuctionEntry[1]) { $nonNegativeEntryFound = true; break; } } if (!$nonNegativeEntryFound) { unset($conjuctions[$conjuctionId]); } } $subqueries = array(); foreach ($conjuctions as $conjuction) { // Check, if it's a one term conjuction if (count($conjuction) == 1) { $subqueries[] = $conjuction[0][0]->getQuery($this->_encoding); } else { $subquery = new Query\Boolean(); foreach ($conjuction as $conjuctionEntry) { $subquery->addSubquery($conjuctionEntry[0]->getQuery($this->_encoding), $conjuctionEntry[1]); } $subqueries[] = $subquery; } } if (count($subqueries) == 0) { return new Query\Insignificant(); } if (count($subqueries) == 1) { return $subqueries[0]; } $query = new Query\Boolean(); foreach ($subqueries as $subquery) { // Non-requirered entry/subquery $query->addSubquery($subquery); } return $query; }
/** * Re-write query into primitive queries in the context of specified index * * @param \ZendSearch\Lucene\SearchIndexInterface $index * @return \ZendSearch\Lucene\Search\Query\AbstractQuery */ public function rewrite(Lucene\SearchIndexInterface $index) { if (count($this->_terms) == 0) { return new EmptyResult(); } elseif ($this->_terms[0]->field !== null) { return $this; } else { $query = new Boolean(); $query->setBoost($this->getBoost()); foreach ($index->getFieldNames(true) as $fieldName) { $subquery = new self(); $subquery->setSlop($this->getSlop()); foreach ($this->_terms as $termId => $term) { $qualifiedTerm = new Index\Term($term->text, $fieldName); $subquery->addTerm($qualifiedTerm, $this->_offsets[$termId]); } $query->addSubquery($subquery); } return $query; } }
/** * Re-write query into primitive queries in the context of specified index * * @param \ZendSearch\Lucene\SearchIndexInterface $index * @return \ZendSearch\Lucene\Search\Query\AbstractQuery */ public function rewrite(Lucene\SearchIndexInterface $index) { // Allow to use wildcards within phrases // They are either removed by text analyzer or used as a part of keyword for keyword fields // // if (strpos($this->_phrase, '?') !== false || strpos($this->_phrase, '*') !== false) { // require_once 'Zend/Search/Lucene/Search/QueryParserException.php'; // throw new Zend_Search_Lucene_Search_QueryParserException('Wildcards are only allowed in a single terms.'); // } // Split query into subqueries if field name is not specified if ($this->_field === null) { $query = new Query\Boolean(); $query->setBoost($this->getBoost()); if (Lucene\Lucene::getDefaultSearchField() === null) { $searchFields = $index->getFieldNames(true); } else { $searchFields = array(Lucene\Lucene::getDefaultSearchField()); } foreach ($searchFields as $fieldName) { $subquery = new Phrase($this->_phrase, $this->_phraseEncoding, $fieldName); $subquery->setSlop($this->getSlop()); $query->addSubquery($subquery->rewrite($index)); } $this->_matches = $query->getQueryTerms(); return $query; } // Recognize exact term matching (it corresponds to Keyword fields stored in the index) // encoding is not used since we expect binary matching $term = new Index\Term($this->_phrase, $this->_field); if ($index->hasTerm($term)) { $query = new Query\Term($term); $query->setBoost($this->getBoost()); $this->_matches = $query->getQueryTerms(); return $query; } // tokenize phrase using current analyzer and process it as a phrase query $tokens = Analyzer::getDefault()->tokenize($this->_phrase, $this->_phraseEncoding); if (count($tokens) == 0) { $this->_matches = array(); return new Query\Insignificant(); } if (count($tokens) == 1) { $term = new Index\Term($tokens[0]->getTermText(), $this->_field); $query = new Query\Term($term); $query->setBoost($this->getBoost()); $this->_matches = $query->getQueryTerms(); return $query; } //It's non-trivial phrase query $position = -1; $query = new Query\Phrase(); foreach ($tokens as $token) { $position += $token->getPositionIncrement(); $term = new Index\Term($token->getTermText(), $this->_field); $query->addTerm($term, $position); $query->setSlop($this->getSlop()); } $this->_matches = $query->getQueryTerms(); return $query; }
/** * Re-write query into primitive queries in the context of specified index * * @param \ZendSearch\Lucene\SearchIndexInterface $index * @throws \ZendSearch\Lucene\Search\Exception\QueryParserException * @return \ZendSearch\Lucene\Search\Query\AbstractQuery */ public function rewrite(Lucene\SearchIndexInterface $index) { if ($this->_field === null) { $query = new Search\Query\Boolean(); $hasInsignificantSubqueries = false; if (Lucene\Lucene::getDefaultSearchField() === null) { $searchFields = $index->getFieldNames(true); } else { $searchFields = array(Lucene\Lucene::getDefaultSearchField()); } foreach ($searchFields as $fieldName) { $subquery = new self($this->_word, $this->_encoding, $fieldName, $this->_minimumSimilarity); $rewrittenSubquery = $subquery->rewrite($index); if (!($rewrittenSubquery instanceof Query\Insignificant || $rewrittenSubquery instanceof Query\EmptyResult)) { $query->addSubquery($rewrittenSubquery); } if ($rewrittenSubquery instanceof Query\Insignificant) { $hasInsignificantSubqueries = true; } } $subqueries = $query->getSubqueries(); if (count($subqueries) == 0) { $this->_matches = array(); if ($hasInsignificantSubqueries) { return new Query\Insignificant(); } else { return new Query\EmptyResult(); } } if (count($subqueries) == 1) { $query = reset($subqueries); } $query->setBoost($this->getBoost()); $this->_matches = $query->getQueryTerms(); return $query; } // ------------------------------------- // Recognize exact term matching (it corresponds to Keyword fields stored in the index) // encoding is not used since we expect binary matching $term = new Index\Term($this->_word, $this->_field); if ($index->hasTerm($term)) { $query = new Query\Fuzzy($term, $this->_minimumSimilarity); $query->setBoost($this->getBoost()); // Get rewritten query. Important! It also fills terms matching container. $rewrittenQuery = $query->rewrite($index); $this->_matches = $query->getQueryTerms(); return $rewrittenQuery; } // ------------------------------------- // Recognize wildcard queries /** * @todo check for PCRE unicode support may be performed through Zend_Environment in some future */ ErrorHandler::start(E_WARNING); $result = preg_match('/\\pL/u', 'a'); ErrorHandler::stop(); if ($result == 1) { $subPatterns = preg_split('/[*?]/u', iconv($this->_encoding, 'UTF-8', $this->_word)); } else { $subPatterns = preg_split('/[*?]/', $this->_word); } if (count($subPatterns) > 1) { throw new QueryParserException('Fuzzy search doesn\'t support wildcards (except within Keyword fields).'); } // ------------------------------------- // Recognize one-term multi-term and "insignificant" queries $tokens = Analyzer\Analyzer::getDefault()->tokenize($this->_word, $this->_encoding); if (count($tokens) == 0) { $this->_matches = array(); return new Query\Insignificant(); } if (count($tokens) == 1) { $term = new Index\Term($tokens[0]->getTermText(), $this->_field); $query = new Query\Fuzzy($term, $this->_minimumSimilarity); $query->setBoost($this->getBoost()); // Get rewritten query. Important! It also fills terms matching container. $rewrittenQuery = $query->rewrite($index); $this->_matches = $query->getQueryTerms(); return $rewrittenQuery; } // Word is tokenized into several tokens throw new QueryParserException('Fuzzy search is supported only for non-multiple word terms'); }