/**
* Search in sphinx client
*
* @param string $query
* @param string $index
* @return SphinxResult|string
* @throws \Exception
*/
public function search($query, $index = '*')
{
$result = $this->_sphinx_client->Query($query, $index);
if (!$result) {
throw new \Exception("Sphinx client error: " . $this->_sphinx_client->GetLastError());
} else {
if (!empty($result['warning'])) {
return $this->_sphinx_client->GetLastWarning();
}
return new SphinxResult($result);
}
}
/**
*
* @param string $query
* @return array of integers - taskIds
*/
public static function searchTasks($query)
{
$fieldWeights = array('description' => 10, 'note' => 6);
$indexName = 'plancake_tasks';
$client = new SphinxClient();
// $client->SetServer (sfConfig::get('app_sphinx_host'), sfConfig::get('app_sphinx_port'));
$client->SetFilter("author_id", array(PcUserPeer::getLoggedInUser()->getId()));
$client->SetConnectTimeout(1);
$client->SetMatchMode(SPH_MATCH_ANY);
$client->SetSortMode(SPH_SORT_RELEVANCE);
$client->SetRankingMode(SPH_RANK_PROXIMITY_BM25);
$client->SetArrayResult(true);
$client->SetFieldWeights($fieldWeights);
$client->setLimits(0, 100);
$results = $client->query($client->EscapeString($query), $indexName);
if ($results === false) {
$error = "Sphinx Error - " . $client->GetLastError();
sfErrorNotifier::alert($error);
}
$ids = array();
if (isset($results['matches']) && count($results['matches'])) {
foreach ($results['matches'] as $match) {
$ids[] = $match['id'];
}
}
return PcTaskPeer::retrieveByPKs($ids);
}
/**
* @{inheritDoc}
*/
public function search()
{
$result = $this->get_result();
// Could be connection to localhost:9312 failed (errno=111,
// msg=Connection refused) during rotate, retry if so
$retries = self::CONNECT_RETRIES;
while (!$result && strpos($this->client->GetLastError(), "errno=111,") !== false && $retries--) {
usleep(self::CONNECT_WAIT_TIME);
$result = $this->get_result();
}
if ($this->client->GetLastError()) {
if ($this->auth->acl_get('a_')) {
throw new http_exception(500, 'SPHINX_SEARCH_FAILED', array($this->client->GetLastError()));
} else {
throw new http_exception(500, 'SPHINX_SEARCH_FAILED_LOG');
}
}
$_result = array('documents' => array(), 'user_ids' => array(), 'total' => 0);
if (!empty($result['matches'])) {
foreach ($result['matches'] as $data) {
$attrs = $data['attrs'];
$attrs['id'] = $attrs['real_id'];
unset($attrs['real_id']);
$_result['documents'][$attrs['type'] . '_' . $attrs['id']] = $attrs;
$_result['user_ids'][] = $attrs['author'];
}
$_result['total'] = $result['total_found'];
}
return $_result;
}
/**
* Adds new integer values set filter to the existing list of filters.
*
* @param $attribute An attribute name.
* @param $values Plain array of integer values.
* @param bool $exclude If set to TRUE, matching items are excluded from the result set.
* @return SphinxSearch_Abstract_List
* @throws Exception on failure
*/
public function setFilter($attribute, $values, $exclude = false)
{
$result = $this->SphinxClient->SetFilter($attribute, $values, $exclude);
if ($result === false) {
throw new Exception("Error on setting filter \"" . $attribute . "\":\n" . $this->SphinxClient->GetLastError());
}
return $this;
}
/**
* {@inheritdoc}
*/
public function query($query, $offset, $perPage, SearchEngineOptions $options = null)
{
if (null === $options) {
$options = new SearchEngineOptions();
}
$this->applyOptions($options);
assert(is_int($offset));
assert($offset >= 0);
assert(is_int($perPage));
$query = $this->parseQuery($query);
$preg = preg_match('/\\s?(recordid|storyid)\\s?=\\s?([0-9]+)/i', $query, $matches, 0, 0);
if ($preg > 0) {
$this->sphinx->SetFilter('record_id', [$matches[2]]);
$query = '';
}
$this->sphinx->SetLimits($offset, $perPage);
$this->sphinx->SetMatchMode(SPH_MATCH_EXTENDED2);
$index = $this->getQueryIndex($query, $options);
$res = $this->sphinx->Query($query, $index);
$results = new ArrayCollection();
if ($res === false) {
if ($this->sphinx->IsConnectError() === true) {
$error = $this->app->trans('Sphinx server is offline');
} else {
$error = $this->sphinx->GetLastError();
}
$warning = $this->sphinx->GetLastWarning();
$total = $available = $duration = 0;
$suggestions = new ArrayCollection();
$propositions = [];
} else {
$error = $res['error'];
$warning = $res['warning'];
$duration = $res['time'];
$total = $res['total_found'];
$available = $res['total'];
$resultOffset = $offset;
if (isset($res['matches'])) {
foreach ($res['matches'] as $record_id => $match) {
try {
$record = new \record_adapter($this->app, $match['attrs']['sbas_id'], $match['attrs']['record_id'], $resultOffset);
$results->add($record);
} catch (\Exception $e) {
}
$resultOffset++;
}
}
$suggestions = $this->getSuggestions($query, $options);
$propositions = '';
}
return new SearchEngineResult($results, $query, $duration, $offset, $available, $total, $error, $warning, $suggestions, $propositions, $index);
}
protected function execute()
{
$sph = $this->sphinxClient->RunQueries();
if ($error = $this->sphinxClient->GetLastError()) {
throw new ESphinxException($error);
}
if ($error = $this->sphinxClient->GetLastWarning()) {
throw new ESphinxException($error);
}
if (!is_array($sph)) {
throw new ESphinxException("Sphinx client returns result not array");
}
$results = array();
foreach ($sph as $result) {
if (isset($result['error']) && strlen($result['error'])) {
throw new ESphinxException($result['error']);
}
$results[] = new ESphinxResult($result);
}
return $results;
}
if ($distinct) {
$cl->SetGroupDistinct($distinct);
}
if ($select) {
$cl->SetSelect($select);
}
if ($limit) {
$cl->SetLimits(0, $limit, $limit > 1000 ? $limit : 1000);
}
$cl->SetRankingMode($ranker);
$res = $cl->Query($q, $index);
////////////////
// print me out
////////////////
if ($res === false) {
print "Query failed: " . $cl->GetLastError() . ".\n";
} else {
if ($cl->GetLastWarning()) {
print "WARNING: " . $cl->GetLastWarning() . "\n\n";
}
print "Query '{$q}' retrieved {$res['total']} of {$res['total_found']} matches in {$res['time']} sec.\n";
print "Query stats:\n";
if (is_array($res["words"])) {
foreach ($res["words"] as $word => $info) {
print " '{$word}' found {$info['hits']} times in {$info['docs']} documents\n";
}
}
print "\n";
if (is_array($res["matches"])) {
$n = 1;
print "Matches:\n";
}
} else {
$query = explode(' ', $query);
$q = '(';
foreach ($query as $k => $w) {
if (strlen($w) > 2) {
$q .= $w . ' | ' . $w . '* | *' . $w . '* | *' . $w;
if ($k < count($query) - 1) {
$q .= ' | ';
}
}
}
$q .= ')';
$result = $sphinx->Query($q, 'name' . $GLOBALS['CONFIG']['search_index_prefix']);
if ($result === false) {
print_r("Произошла ошибка: " . $sphinx->GetLastError());
}
$i = 0;
foreach ($result['matches'] as $val) {
$res[$i] = $val['id'];
$i++;
}
if (!empty($res) && count($res > 0)) {
$where_arr['customs'][] = 'p.id_product IN (' . implode(', ', $res) . ')';
} else {
$where_arr['customs'][] = 'p.id_product = 2';
}
}
}
}
// =========================================================
public function run($subject_id, $clean = true, $query_offset = 0, $from, $to)
{
$this->load->helper('sphinxapi');
$this->load->helper('mood');
// skip if matching_status is "matching"
$matching_status = $this->custom_model->get_value('subject', 'matching_status', $subject_id);
if ($matching_status == 'matching') {
echo "subject is matching";
return false;
}
// flag subject as matching.. do other bot runs this queue.
//$this->db->update('subject',array('matching_status'=>'matching'),array('id'=>$subject_id));
// clear all match record for this subject
$config['hostname'] = "192.168.1.102";
$config['username'] = "******";
$config['password'] = "******";
$config['database'] = "thothconnect";
$config['dbdriver'] = "mysql";
$config['dbprefix'] = "";
$config['pconnect'] = FALSE;
$config['db_debug'] = TRUE;
$config['cache_on'] = FALSE;
$config['cachedir'] = "";
$config['char_set'] = "utf8";
$config['dbcollat'] = "utf8_general_ci";
$thothconnect_db = $this->load->database($config, true);
$query = $this->db->query("SELECT client_id FROM subject WHERE id = " . $subject_id);
$row = $query->row();
$client_id = $row->client_id;
if ($clean) {
$thothconnect_db->delete('website_c' . $client_id, array('subject_id' => $subject_id));
$thothconnect_db->delete('twitter_c' . $client_id, array('subject_id' => $subject_id));
$thothconnect_db->delete('facebook_c' . $client_id, array('subject_id' => $subject_id));
}
//
// begin re-matching this subject
//
// get search string from subject_id
$query = $this->custom_model->get_value('subject', 'query', $subject_id);
// sphinx init
$cl = new SphinxClient();
$q = $query;
$sql = "";
$mode = SPH_MATCH_EXTENDED;
$host = "192.168.1.102";
$port = 9312;
$index = "*";
$groupby = "";
$groupsort = "@group desc";
$filter = "group_id";
$filtervals = array();
$distinct = "";
$sortby = "@id ASC";
$sortexpr = "";
$offset = $query_offset;
$limit = 1000000;
$ranker = SPH_RANK_PROXIMITY_BM25;
$select = "";
echo 'limit=' . $limit . ' offset=' . $offset . PHP_EOL;
//Extract subject keyword from search string
$keywords = get_keywords($q);
////////////
// do query
////////////
$cl->SetServer($host, $port);
$cl->SetConnectTimeout(1);
$cl->SetArrayResult(true);
$cl->SetWeights(array(100, 1));
$cl->SetMatchMode($mode);
// if ( count($filtervals) ) $cl->SetFilter ( $filter, $filtervals );
// if ( $groupby ) $cl->SetGroupBy ( $groupby, SPH_GROUPBY_ATTR, $groupsort );
if ($sortby) {
$cl->SetSortMode(SPH_SORT_EXTENDED, $sortby);
}
// if ( $sortexpr ) $cl->SetSortMode ( SPH_SORT_EXPR, $sortexpr );
if ($distinct) {
$cl->SetGroupDistinct($distinct);
}
if ($select) {
$cl->SetSelect($select);
}
if ($limit) {
$cl->SetLimits(0, $limit, $limit > 1000000 ? $limit : 1000000);
}
$cl->SetRankingMode($ranker);
$res = $cl->Query($q, $index);
//$res = true;
////////////
// do Insert to DB
////////////
// Current matching
$current_matching = array();
/*$query_matchs = $this->db->get_where('matchs',array('subject_id'=>$subject_id));
if($query_matchs->num_rows() > 0)
{
echo PHP_EOL.'currents matching :'.$query_matchs->num_rows();
foreach($query_matchs->result() as $match)
{
$current_matching[] = $match->post_id;
}
}*/
// set matching date range from-to
$from = strtotime($from);
$to = strtotime($to);
// Search and Update
if ($res === false) {
echo "Query failed: " . $cl->GetLastError() . ".\n";
} else {
if ($cl->GetLastWarning()) {
echo "WARNING: " . $cl->GetLastWarning() . "\n\n";
}
echo "Query '{$q}' \nretrieved {$res['total']} of {$res['total_found']} matches in {$res['time']} sec.\n";
if ($res['total'] == 0) {
echo "no result<br/>\n";
} else {
if ($res['total'] > $limit + $offset) {
$this->run($subject_id, $limit + $offset);
} else {
echo "Updating...";
foreach ($res["matches"] as $k => $docinfo) {
// echo '('.$k.')'.$docinfo["id"]." ";
// Reset PHP Timeout to 1min
// if found in $current_matching then skip
if (in_array($docinfo["id"], $current_matching)) {
continue;
} else {
// else insert new match
set_time_limit(60);
$post = new Post_model();
$post->init($docinfo["id"]);
// if post_date is our of range then skip
$post_date = strtotime($post->post_date);
if ($post_date < $from || $post_date > $to) {
continue;
}
$mood = get_mood($post->body, $keywords);
//-----------------------------------------------------
$subject = $post->get_subject($subject_id);
//print_r($subject);
if ($post->type == "post" || $post->type == "comment") {
$postData = $post->get_post_website($post->id);
if ($postData != null) {
$data = array();
$data["post_id"] = $postData->post_id;
$data["post_date"] = $postData->post_date;
$data["title"] = $postData->title;
$data["body"] = $postData->body;
$data["type"] = $postData->type;
$data["author_id"] = $postData->author_id;
$data["author"] = $postData->author;
$data["website_id"] = $postData->website_id;
$data["website_name"] = $postData->website_name;
$data["website_cate_id"] = $postData->website_cate_id;
$data["website_cate"] = $postData->website_cate;
$data["website_type_id"] = $postData->website_type_id;
$data["website_type"] = $postData->website_type;
$data["group_id"] = $subject->group_id;
$data["group"] = $subject->group;
$data["url"] = substr($postData->root_url, 0, -1) . "" . $postData->url;
$data["page_id"] = $postData->page_id;
$data["subject_id"] = $subject->subject_id;
$data["subject_name"] = $subject->subject_name;
$data["mood"] = $mood;
$data["mood_by"] = 'system';
$thothconnect_db->insert("website_c" . $subject->client_id, $data);
$post->insert_post_comment($postData->page_id, $subject->client_id, $thothconnect_db);
}
} else {
if ($post->type == "tweet" || $post->type == "retweet") {
$postData = $post->get_post_twitter($post->id);
if ($postData != null) {
$data = array();
$data["post_id"] = $postData->post_id;
$data["post_date"] = $postData->post_date;
$data["body"] = $postData->body;
$data["type"] = $postData->type;
$data["author_id"] = $postData->author_id;
$data["author"] = $postData->author;
$data["group_id"] = $subject->group_id;
$data["group"] = $subject->group;
$data["tweet_id"] = $postData->tweet_id;
$data["subject_id"] = $subject->subject_id;
$data["subject_name"] = $subject->subject_name;
$data["mood"] = $mood;
$data["mood_by"] = 'system';
$thothconnect_db->insert("twitter_c" . $subject->client_id, $data);
}
} else {
if ($post->type == "fb_post" || $post->type == "fb_comment") {
$postData = $post->get_post_facebook($post->id);
if ($postData != null) {
$data = array();
$data["post_id"] = $postData->post_id;
$data["post_date"] = $postData->post_date;
$data["body"] = $postData->body;
$data["type"] = $postData->type;
$data["author_id"] = $postData->author_id;
$data["author"] = $postData->author;
$data["group_id"] = $subject->group_id;
$data["group"] = $subject->group;
$data["facebook_page_id"] = $postData->facebook_page_id;
$data["facebook_page_name"] = $postData->facebook_page_name;
$data["subject_id"] = $subject->subject_id;
$data["subject_name"] = $subject->subject_name;
$data["facebook_id"] = $postData->facebook_id;
$data["parent_post_id"] = $postData->parent_post_id;
$data["likes"] = $postData->likes;
$data["shares"] = $postData->shares;
$data["mood"] = $mood;
$data["mood_by"] = 'system';
$thothconnect_db->insert("facebook_c" . $subject->client_id, $data);
}
}
}
}
/*
$data = array(
'post_id'=> $post->id,
'subject_id' => $subject_id ,
'matching_date' => null,
'sentiment' => $mood,
'by' => 'system',
'system_correct' => $mood,
'system_correct_date' => mdate('%Y-%m-%d %H:%i',time())
);
$this->db->insert('matchs',$data);
*/
//---------------------------------------
}
}
}
}
}
// flag subject as update..
$data = array('matching_status' => 'update', 'latest_matching' => mdate('%Y-%m-%d %H:%i:%s', time()), 'from' => mdate('%Y-%m-%d %H:%i:%s', $from), 'to' => mdate('%Y-%m-%d %H:%i:%s', $to));
$this->db->update('subject', $data, array('id' => $subject_id));
}
function getErrorMessage()
{
return $this->sphinx->GetLastError();
}
//---------------BEGIN SPHINX
require_once $basePath . "/../SPHINX.class.php";
// Get the search variable from URL
// $var = @$_GET['msg_mask'] ;
// $trimmed = trim($$msg_mask); //trim whitespace from the stored variable
// $q = $trimmed;
#$q = "SELECT id ,group_id,title FROM documents where title = 'test one'";
#$q = " SELECT id, group_id, UNIX_TIMESTAMP(date_added) AS date_added, title, content FROM documents";
$index = "idx_logs";
$cl = new SphinxClient();
$hostip = $_SESSION['SPX_SRV'];
$port = intval($_SESSION['SPX_PORT']);
$cl->SetServer($hostip, $port);
$res = $cl->Query($msg_mask, $index);
if (!$res) {
die("ERROR: " . $cl->GetLastError() . ".\n");
} else {
if ($res['total_found'] > 0) {
$where .= " AND id IN (";
foreach ($res["matches"] as $doc => $docinfo) {
$where .= "'{$doc}',";
// echo "$doc<br>\n";
}
$where = rtrim($where, ",");
$where .= ")";
} else {
// Negate search since sphinx returned 0 hits
$where = "WHERE 1<1";
// die(print_r($res));
}
}
/**
* Return last error
* @param bool $conn
* @return bool|string
*/
public function getError($conn = false)
{
return !$conn ? $this->client->GetLastError() : $this->client->IsConnectError();
}
function sphinx_search_action($arrSearch)
{
global $PHORUM;
// No pecl class, try php version
if (!class_exists('SphinxClient')) {
// loads from php include_path
require_once 'sphinxapi.php';
}
// these are the index-names set in sphinx.conf - one for searching messages, the other for searching by authors only
// both contain an additional index for the deltas - changes done after the last full reindex
$index_name_msg = 'phorum5_msg_d phorum5_msg';
$index_name_author = 'phorum5_author phorum5_author_d';
// excerpts_index is just one index as that function only accepts one, it used for determining charsets / mapping tables, nothing more
$excerpts_index = 'phorum5_msg';
$index = $index_name_msg;
if ($arrSearch['match_type'] == 'ALL') {
$match_mode = SPH_MATCH_ALL;
} elseif ($arrSearch['match_type'] == 'ANY') {
$match_mode = SPH_MATCH_ANY;
} elseif ($arrSearch['match_type'] == 'PHRASE') {
$match_mode = SPH_MATCH_PHRASE;
} elseif ($arrSearch['match_type'] == 'AUTHOR') {
$match_mode = SPH_MATCH_PHRASE;
$index = $index_name_author;
} else {
// Return search control to Phorum in case the search type isn't handled by the module.
return $arrSearch;
}
if (empty($arrSearch['search']) && !empty($arrSearch['author'])) {
$arrSearch['search'] = $arrSearch['author'];
$index = $index_name_author;
}
$sphinx = new SphinxClient();
$sphinx->SetServer($PHORUM['mod_sphinx_search']['hostname'], $PHORUM['mod_sphinx_search']['port']);
$sphinx->SetMatchMode($match_mode);
// set the limits for paging
$sphinx->SetLimits($arrSearch['offset'], $arrSearch['length']);
// set the timeframe to search
if ($arrSearch['match_dates'] > 0) {
$min_ts = time() - 86400 * $arrSearch['match_dates'];
$max_ts = time();
$sphinx->SetFilterRange('datestamp', $min_ts, $max_ts);
}
// Check what forums the active Phorum user can read.
$allowed_forums = phorum_api_user_check_access(PHORUM_USER_ALLOW_READ, PHORUM_ACCESS_LIST);
// If the user is not allowed to search any forum or the current
// active forum, then return the emtpy search results array.
if (empty($allowed_forums) || $PHORUM['forum_id'] > 0 && !in_array($PHORUM['forum_id'], $allowed_forums)) {
$arrSearch['results'] = array();
$arrSearch['totals'] = 0;
$arrSearch['continue'] = 0;
$arrSearch['raw_body'] = 1;
return $arrSearch;
}
// Prepare forum_id restriction.
$search_forums = array();
foreach (explode(',', $arrSearch['match_forum']) as $forum_id) {
if ($forum_id == 'ALL') {
$search_forums = $allowed_forums;
break;
}
if (isset($allowed_forums[$forum_id])) {
$search_forums[] = $forum_id;
}
}
$sphinx->SetFilter('forum_id', $search_forums);
// set the sort-mode
$sphinx->SetSortMode(SPH_SORT_ATTR_DESC, 'datestamp');
// do the actual query
$results = $sphinx->Query($arrSearch['search'], $index);
$res = $sphinx->GetLastWarning();
if ($res) {
error_log("sphinx_search.php: WARNING: {$res}");
}
$res = $sphinx->GetLastError();
if ($res) {
error_log("sphinx_search.php: ERROR: {$res}");
}
// if no messages were found, then return empty handed.
if (!isset($results['matches'])) {
$arrSearch['results'] = array();
$arrSearch['totals'] = 0;
$arrSearch['continue'] = 0;
$arrSearch['raw_body'] = 1;
return $arrSearch;
}
$search_msg_ids = $results['matches'];
// get the messages we found
$found_messages = phorum_db_get_message(array_keys($search_msg_ids), 'message_id', true);
// sort them in reverse order of the message_id to automagically sort them by date desc this way
krsort($found_messages);
reset($found_messages);
// prepare the array for building highlighted excerpts
$docs = array();
foreach ($found_messages as $id => $data) {
// remove hidden text in the output - only added by the hidden_msg module
$data['body'] = preg_replace("/(\\[hide=([\\#a-z0-9]+?)\\](.+?)\\[\\/hide\\])/is", '', $data['body']);
$docs[] = htmlspecialchars(phorum_strip_body($data['body']));
}
$words = '';
if (!empty($results['words'])) {
$words = implode(' ', array_keys($results['words']));
}
$opts = array('chunk_separator' => ' [...] ');
// build highlighted excerpts
$highlighted = $sphinx->BuildExcerpts($docs, $excerpts_index, $words, $opts);
$res = $sphinx->GetLastWarning();
if ($res) {
error_log("sphinx_search.php: WARNING: {$res}");
}
$res = $sphinx->GetLastError();
if ($res) {
error_log("sphinx_search.php: ERROR: {$res}");
}
$cnt = 0;
foreach ($found_messages as $id => $content) {
$found_messages[$id]['short_body'] = $highlighted[$cnt];
$cnt++;
}
$arrSearch['results'] = $found_messages;
// we need the total results
$arrSearch['totals'] = $results['total_found'];
if ($arrSearch['totals'] > 1000) {
$arrSearch['totals'] = 1000;
}
// don't run the default search
$arrSearch['continue'] = 0;
// tell it to leave the body alone
$arrSearch['raw_body'] = 1;
return $arrSearch;
}
public function __construct($rowsPerPage, $currentPage, $siteID, $wildCardString, $sortBy, $sortDirection)
{
$this->_db = DatabaseConnection::getInstance();
$this->_siteID = $siteID;
$this->_sortByFields = array('firstName', 'lastName', 'city', 'state', 'dateModifiedSort', 'dateCreatedSort', 'ownerSort');
if (ENABLE_SPHINX) {
/* Sphinx API likes to throw PHP errors *AND* use it's own error
* handling.
*/
assert_options(ASSERT_WARNING, 0);
$sphinx = new SphinxClient();
$sphinx->SetServer(SPHINX_HOST, SPHINX_PORT);
$sphinx->SetWeights(array(0, 100, 0, 0, 50));
$sphinx->SetMatchMode(SPH_MATCH_EXTENDED);
$sphinx->SetLimits(0, 1000);
$sphinx->SetSortMode(SPH_SORT_TIME_SEGMENTS, 'date_added');
// FIXME: This can be sped up a bit by actually grouping ranges of
// site IDs into their own index's. Maybe every 500 or so at
// least on the Hosted system.
$sphinx->SetFilter('site_id', array($this->_siteID));
/* Create the Sphinx query string. */
$wildCardString = DatabaseSearch::humanToSphinxBoolean($wildCardString);
/* Execute the Sphinx query. Sphinx can ask us to retry if its
* maxed out. Retry up to 5 times.
*/
$tries = 0;
do {
/* Wait for one second if this isn't out first attempt. */
if (++$tries > 1) {
sleep(1);
}
$results = $sphinx->Query($wildCardString, SPHINX_INDEX);
$errorMessage = $sphinx->GetLastError();
} while ($results === false && strpos($errorMessage, 'server maxed out, retry') !== false && $tries <= 5);
/* Throw a fatal error if Sphinx errors occurred. */
if ($results === false) {
$this->fatal('Sphinx Error: ' . ucfirst($errorMessage) . '.');
}
/* Throw a fatal error (for now) if Sphinx warnings occurred. */
$lastWarning = $sphinx->GetLastWarning();
if (!empty($lastWarning)) {
// FIXME: Just display a warning, and notify dev team.
$this->fatal('Sphinx Warning: ' . ucfirst($lastWarning) . '.');
}
/* Show warnings for assert()s again. */
assert_options(ASSERT_WARNING, 1);
if (empty($results['matches'])) {
$this->_WHERE = '0';
} else {
$attachmentIDs = implode(',', array_keys($results['matches']));
$this->_WHERE = 'attachment.attachment_id IN(' . $attachmentIDs . ')';
}
} else {
$this->_WHERE = DatabaseSearch::makeBooleanSQLWhere(DatabaseSearch::fulltextEncode($wildCardString), $this->_db, 'attachment.text');
}
/* How many companies do we have? */
$sql = sprintf("SELECT\n COUNT(*) AS count\n FROM\n attachment\n LEFT JOIN candidate\n ON attachment.data_item_id = candidate.candidate_id\n AND attachment.data_item_type = %s\n AND attachment.site_id = candidate.site_id\n LEFT JOIN user AS owner_user\n ON candidate.owner = owner_user.user_id\n WHERE\n resume = 1\n AND\n %s\n AND\n (ISNULL(candidate.is_admin_hidden) OR (candidate.is_admin_hidden = 0))\n AND\n (ISNULL(candidate.is_active) OR (candidate.is_active = 1))\n AND\n attachment.site_id = %s", DATA_ITEM_CANDIDATE, $this->_WHERE, $this->_siteID);
$rs = $this->_db->getAssoc($sql);
/* Pass "Search By Resume"-specific parameters to Pager constructor. */
parent::__construct($rs['count'], $rowsPerPage, $currentPage);
}
/**
* Performs a search on keywords depending on display specific params. You have to run split_keywords() first
*
* @param string $type contains either posts or topics depending on what should be searched for
* @param string $fields contains either titleonly (topic titles should be searched), msgonly (only message bodies should be searched), firstpost (only subject and body of the first post should be searched) or all (all post bodies and subjects should be searched)
* @param string $terms is either 'all' (use query as entered, words without prefix should default to "have to be in field") or 'any' (ignore search query parts and just return all posts that contain any of the specified words)
* @param array $sort_by_sql contains SQL code for the ORDER BY part of a query
* @param string $sort_key is the key of $sort_by_sql for the selected sorting
* @param string $sort_dir is either a or d representing ASC and DESC
* @param string $sort_days specifies the maximum amount of days a post may be old
* @param array $ex_fid_ary specifies an array of forum ids which should not be searched
* @param string $post_visibility specifies which types of posts the user can view in which forums
* @param int $topic_id is set to 0 or a topic id, if it is not 0 then only posts in this topic should be searched
* @param array $author_ary an array of author ids if the author should be ignored during the search the array is empty
* @param string $author_name specifies the author match, when ANONYMOUS is also a search-match
* @param array &$id_ary passed by reference, to be filled with ids for the page specified by $start and $per_page, should be ordered
* @param int $start indicates the first index of the page
* @param int $per_page number of ids each page is supposed to contain
* @return boolean|int total number of results
*/
public function keyword_search($type, $fields, $terms, $sort_by_sql, $sort_key, $sort_dir, $sort_days, $ex_fid_ary, $post_visibility, $topic_id, $author_ary, $author_name, &$id_ary, &$start, $per_page)
{
global $user, $phpbb_log;
// No keywords? No posts.
if (!strlen($this->search_query) && !sizeof($author_ary)) {
return false;
}
$id_ary = array();
// Sorting
if ($type == 'topics') {
switch ($sort_key) {
case 'a':
$this->sphinx->SetGroupBy('topic_id', SPH_GROUPBY_ATTR, 'poster_id ' . ($sort_dir == 'a' ? 'ASC' : 'DESC'));
break;
case 'f':
$this->sphinx->SetGroupBy('topic_id', SPH_GROUPBY_ATTR, 'forum_id ' . ($sort_dir == 'a' ? 'ASC' : 'DESC'));
break;
case 'i':
case 's':
$this->sphinx->SetGroupBy('topic_id', SPH_GROUPBY_ATTR, 'post_subject ' . ($sort_dir == 'a' ? 'ASC' : 'DESC'));
break;
case 't':
default:
$this->sphinx->SetGroupBy('topic_id', SPH_GROUPBY_ATTR, 'topic_last_post_time ' . ($sort_dir == 'a' ? 'ASC' : 'DESC'));
break;
}
} else {
switch ($sort_key) {
case 'a':
$this->sphinx->SetSortMode($sort_dir == 'a' ? SPH_SORT_ATTR_ASC : SPH_SORT_ATTR_DESC, 'poster_id');
break;
case 'f':
$this->sphinx->SetSortMode($sort_dir == 'a' ? SPH_SORT_ATTR_ASC : SPH_SORT_ATTR_DESC, 'forum_id');
break;
case 'i':
case 's':
$this->sphinx->SetSortMode($sort_dir == 'a' ? SPH_SORT_ATTR_ASC : SPH_SORT_ATTR_DESC, 'post_subject');
break;
case 't':
default:
$this->sphinx->SetSortMode($sort_dir == 'a' ? SPH_SORT_ATTR_ASC : SPH_SORT_ATTR_DESC, 'post_time');
break;
}
}
// Most narrow filters first
if ($topic_id) {
$this->sphinx->SetFilter('topic_id', array($topic_id));
}
/**
* Allow modifying the Sphinx search options
*
* @event core.search_sphinx_keywords_modify_options
* @var string type Searching type ('posts', 'topics')
* @var string fields Searching fields ('titleonly', 'msgonly', 'firstpost', 'all')
* @var string terms Searching terms ('all', 'any')
* @var int sort_days Time, in days, of the oldest possible post to list
* @var string sort_key The sort type used from the possible sort types
* @var int topic_id Limit the search to this topic_id only
* @var array ex_fid_ary Which forums not to search on
* @var string post_visibility Post visibility data
* @var array author_ary Array of user_id containing the users to filter the results to
* @var string author_name The username to search on
* @var object sphinx The Sphinx searchd client object
* @since 3.1.7-RC1
*/
$sphinx = $this->sphinx;
$vars = array('type', 'fields', 'terms', 'sort_days', 'sort_key', 'topic_id', 'ex_fid_ary', 'post_visibility', 'author_ary', 'author_name', 'sphinx');
extract($this->phpbb_dispatcher->trigger_event('core.search_sphinx_keywords_modify_options', compact($vars)));
$this->sphinx = $sphinx;
unset($sphinx);
$search_query_prefix = '';
switch ($fields) {
case 'titleonly':
// Only search the title
if ($terms == 'all') {
$search_query_prefix = '@title ';
}
// Weight for the title
$this->sphinx->SetFieldWeights(array("title" => 5, "data" => 1));
// 1 is first_post, 0 is not first post
$this->sphinx->SetFilter('topic_first_post', array(1));
break;
case 'msgonly':
// Only search the body
if ($terms == 'all') {
$search_query_prefix = '@data ';
}
// Weight for the body
$this->sphinx->SetFieldWeights(array("title" => 1, "data" => 5));
break;
case 'firstpost':
// More relative weight for the title, also search the body
$this->sphinx->SetFieldWeights(array("title" => 5, "data" => 1));
// 1 is first_post, 0 is not first post
$this->sphinx->SetFilter('topic_first_post', array(1));
break;
default:
// More relative weight for the title, also search the body
$this->sphinx->SetFieldWeights(array("title" => 5, "data" => 1));
break;
}
if (sizeof($author_ary)) {
$this->sphinx->SetFilter('poster_id', $author_ary);
}
// As this is not simply possible at the moment, we limit the result to approved posts.
// This will make it impossible for moderators to search unapproved and softdeleted posts,
// but at least it will also cause the same for normal users.
$this->sphinx->SetFilter('post_visibility', array(ITEM_APPROVED));
if (sizeof($ex_fid_ary)) {
// All forums that a user is allowed to access
$fid_ary = array_unique(array_intersect(array_keys($this->auth->acl_getf('f_read', true)), array_keys($this->auth->acl_getf('f_search', true))));
// All forums that the user wants to and can search in
$search_forums = array_diff($fid_ary, $ex_fid_ary);
if (sizeof($search_forums)) {
$this->sphinx->SetFilter('forum_id', $search_forums);
}
}
$this->sphinx->SetFilter('deleted', array(0));
$this->sphinx->SetLimits($start, (int) $per_page, SPHINX_MAX_MATCHES);
$result = $this->sphinx->Query($search_query_prefix . str_replace('"', '"', $this->search_query), $this->indexes);
// Could be connection to localhost:9312 failed (errno=111,
// msg=Connection refused) during rotate, retry if so
$retries = SPHINX_CONNECT_RETRIES;
while (!$result && strpos($this->sphinx->GetLastError(), "errno=111,") !== false && $retries--) {
usleep(SPHINX_CONNECT_WAIT_TIME);
$result = $this->sphinx->Query($search_query_prefix . str_replace('"', '"', $this->search_query), $this->indexes);
}
if ($this->sphinx->GetLastError()) {
$phpbb_log->add('critical', $user->data['user_id'], $user->ip, 'LOG_SPHINX_ERROR', false, array($this->sphinx->GetLastError()));
if ($this->auth->acl_get('a_')) {
trigger_error($this->user->lang('SPHINX_SEARCH_FAILED', $this->sphinx->GetLastError()));
} else {
trigger_error($this->user->lang('SPHINX_SEARCH_FAILED_LOG'));
}
}
$result_count = $result['total_found'];
if ($result_count && $start >= $result_count) {
$start = floor(($result_count - 1) / $per_page) * $per_page;
$this->sphinx->SetLimits((int) $start, (int) $per_page, SPHINX_MAX_MATCHES);
$result = $this->sphinx->Query($search_query_prefix . str_replace('"', '"', $this->search_query), $this->indexes);
// Could be connection to localhost:9312 failed (errno=111,
// msg=Connection refused) during rotate, retry if so
$retries = SPHINX_CONNECT_RETRIES;
while (!$result && strpos($this->sphinx->GetLastError(), "errno=111,") !== false && $retries--) {
usleep(SPHINX_CONNECT_WAIT_TIME);
$result = $this->sphinx->Query($search_query_prefix . str_replace('"', '"', $this->search_query), $this->indexes);
}
}
$id_ary = array();
if (isset($result['matches'])) {
if ($type == 'posts') {
$id_ary = array_keys($result['matches']);
} else {
foreach ($result['matches'] as $key => $value) {
$id_ary[] = $value['attrs']['topic_id'];
}
}
} else {
return false;
}
$id_ary = array_slice($id_ary, 0, (int) $per_page);
return $result_count;
}
public function run($subject_id, $clean = true, $query_offset = 0, $from, $to)
{
$this->load->helper('sphinxapi');
$this->load->helper('mood');
// skip if matching_status is "matching"
$matching_status = $this->custom_model->get_value('subject', 'matching_status', $subject_id);
if ($matching_status == 'matching') {
echo "subject is matching";
return false;
}
// flag subject as matching.. do other bot runs this queue.
$this->db->update('subject', array('matching_status' => 'matching'), array('id' => $subject_id));
// clear all match record for this subject
if ($clean) {
$this->db->delete('matchs', array('subject_id' => $subject_id));
}
//
// begin re-matching this subject
//
// get search string from subject_id
$query = $this->custom_model->get_value('subject', 'query', $subject_id);
// sphinx init
$cl = new SphinxClient();
$q = $query;
$sql = "";
$mode = SPH_MATCH_EXTENDED;
$host = "192.168.1.102";
$port = 9312;
$index = "*";
$groupby = "";
$groupsort = "@group desc";
$filter = "group_id";
$filtervals = array();
$distinct = "";
$sortby = "@id ASC";
$sortexpr = "";
$offset = $query_offset;
$limit = 1000000;
$ranker = SPH_RANK_PROXIMITY_BM25;
$select = "";
echo 'limit=' . $limit . ' offset=' . $offset . PHP_EOL;
//Extract subject keyword from search string
$keywords = get_keywords($q);
////////////
// do query
////////////
$cl->SetServer($host, $port);
$cl->SetConnectTimeout(1);
$cl->SetArrayResult(true);
$cl->SetWeights(array(100, 1));
$cl->SetMatchMode($mode);
// if ( count($filtervals) ) $cl->SetFilter ( $filter, $filtervals );
// if ( $groupby ) $cl->SetGroupBy ( $groupby, SPH_GROUPBY_ATTR, $groupsort );
if ($sortby) {
$cl->SetSortMode(SPH_SORT_EXTENDED, $sortby);
}
// if ( $sortexpr ) $cl->SetSortMode ( SPH_SORT_EXPR, $sortexpr );
if ($distinct) {
$cl->SetGroupDistinct($distinct);
}
if ($select) {
$cl->SetSelect($select);
}
if ($limit) {
$cl->SetLimits(0, $limit, $limit > 1000000 ? $limit : 1000000);
}
$cl->SetRankingMode($ranker);
$res = $cl->Query($q, $index);
////////////
// do Insert to DB
////////////
// Current matching
$current_matching = array();
$query_matchs = $this->db->get_where('matchs', array('subject_id' => $subject_id));
if ($query_matchs->num_rows() > 0) {
echo PHP_EOL . 'currents matching :' . $query_matchs->num_rows();
foreach ($query_matchs->result() as $match) {
$current_matching[] = $match->post_id;
}
}
// set matching date range from-to
$from = strtotime($from);
$to = strtotime($to);
// Search and Update
if ($res === false) {
echo "Query failed: " . $cl->GetLastError() . ".\n";
} else {
if ($cl->GetLastWarning()) {
echo "WARNING: " . $cl->GetLastWarning() . "\n\n";
}
echo "Query '{$q}' \nretrieved {$res['total']} of {$res['total_found']} matches in {$res['time']} sec.\n";
if ($res['total'] == 0) {
echo "no result<br/>\n";
} else {
if ($res['total'] > $limit + $offset) {
$this->run($subject_id, $limit + $offset);
} else {
echo "Updating...";
foreach ($res["matches"] as $k => $docinfo) {
// echo '('.$k.')'.$docinfo["id"]." ";
// Reset PHP Timeout to 1min
// if found in $current_matching then skip
if (in_array($docinfo["id"], $current_matching)) {
continue;
} else {
// else insert new match
set_time_limit(60);
$post = new Post_model();
$post->init($docinfo["id"]);
// if post_date is our of range then skip
$post_date = strtotime($post->post_date);
if ($post_date < $from || $post_date > $to) {
continue;
}
$mood = get_mood($post->body, $keywords);
$data = array('post_id' => $post->id, 'subject_id' => $subject_id, 'matching_date' => null, 'sentiment' => $mood, 'by' => 'system', 'system_correct' => $mood, 'system_correct_date' => mdate('%Y-%m-%d %H:%i', time()));
$this->db->insert('matchs', $data);
}
}
}
}
}
// flag subject as update..
$data = array('matching_status' => 'update', 'latest_matching' => mdate('%Y-%m-%d %H:%i:%s', time()), 'from' => mdate('%Y-%m-%d %H:%i:%s', $from), 'to' => mdate('%Y-%m-%d %H:%i:%s', $to), 'bot_id' => 0);
$this->db->update('subject', $data, array('id' => $subject_id));
}
public static function queryDocument($query, $params = array())
{
if (trim($query) == "") {
return array();
}
$sphinx_config = SphinxSearch_Config::getInstance();
$documents_config = $sphinx_config->getDocumentsAsArray();
$SphinxClient = new SphinxClient();
$entries = array();
$language = "all";
if (SphinxSearch_Config_Plugin::getValue("documents", "use_i18n") == "true") {
if (array_key_exists("language", $params)) {
$language = $params["language"];
} else {
$locale = Zend_Registry::get("Zend_Locale");
$language = $locale->getLanguage();
}
}
$field_weights = array();
$indexes = array();
foreach ($documents_config as $document_name => $document_properties) {
$indexes[] = "idx_document_" . $document_name . "_" . $language;
foreach ($document_properties["elements"] as $field_name => $field_config) {
if (array_key_exists("weight", $field_config) && intval($field_config["weight"]) > 0) {
$field_weights[$field_name] = intval($field_config["weight"]);
}
}
}
if (sizeof($field_weights) > 0) {
$SphinxClient->setFieldWeights($field_weights);
}
$search_result = $SphinxClient->Query($query, implode(", ", $indexes));
if ($search_result === false) {
throw new Exception($SphinxClient->GetLastError());
}
if ($search_result["total_found"] > 0) {
foreach ($search_result["matches"] as $id => $meta) {
$entries[] = array("result" => Document::getById($id), "id" => $id, "meta" => $meta, "type" => "document");
}
}
return $entries;
}
/**
* Получить ошибку при последнем обращении к поиску
*
* @return string
*/
public function GetLastError()
{
return mb_convert_encoding($this->oSphinx->GetLastError(), 'UTF-8');
}
protected function QuerySearch($index, $sort, $order, $port = 9312, $host = 'localhost')
{
$sphinx = new SphinxClient();
$sphinx->SetServer($host, $port);
$sphinx->SetConnectTimeout(1);
$sphinx->SetArrayResult(true);
$sphinx->SetLimits(0, static::LIMIT_SEARCH);
$sphinx->SetMatchMode(SPH_MATCH_EXTENDED2);
$sphinx->SetSortMode($order, $sort);
// Limit results to a certain period
if (!is_null($this->search_range)) {
$sphinx->SetFilterRange($sort, time() - $this->search_range, time());
}
// Check for multi-query search
if (is_array($this->search)) {
foreach ($this->search as $query) {
if (!empty($query)) {
$sphinx->AddQuery($this->FilterSearch($query), $index);
}
}
$result = $sphinx->RunQueries();
} else {
$result = $sphinx->Query($this->FilterSearch($this->search), $index);
}
if ($result === false) {
throw new ErrorException('Search failed: ' . $sphinx->GetLastError());
}
// Return result
$ids = array();
$ids[] = 0;
// Make IN() valid even if Sphinx returned nothing
if (is_array($this->search)) {
// Merge results from multi-query search
foreach ($result as $r) {
if (isset($r['matches'])) {
foreach ($r['matches'] as $match) {
$ids[] = $match['id'];
}
}
}
} elseif (isset($result['matches'])) {
foreach ($result['matches'] as $match) {
$ids[] = $match['id'];
}
}
return $ids;
}
<?php
/**
* User: yunsong
* Date: 3/12/16
* Time: 10:46 AM
* Email:awebc@qq.com
*/
require "../../common/components/SphinxClient.php";
$sphinx = new SphinxClient();
$sphinx->SetServer("127.0.0.1", 9312);
// $sphinx->SetMatchMode(6);// SPH_MATCH_EXTENDED2
$sphinx->SetSelect("id");
$sphinx->SetArrayResult(true);
$sphinx->SetLimits(0, 20);
$res = $sphinx->Query($query, "feeds");
echo '<pre>';
print_r($res['matches']);
print_r($res);
print_r($sphinx->GetLastError());
print_r($sphinx->GetLastWarning());
echo '</pre>';
echo 1;
/**
* Performs a search on keywords depending on display specific params. You have to run split_keywords() first
*
* @param string $type contains either posts or topics depending on what should be searched for
* @param string $fields contains either titleonly (topic titles should be searched), msgonly (only message bodies should be searched), firstpost (only subject and body of the first post should be searched) or all (all post bodies and subjects should be searched)
* @param string $terms is either 'all' (use query as entered, words without prefix should default to "have to be in field") or 'any' (ignore search query parts and just return all posts that contain any of the specified words)
* @param array $sort_by_sql contains SQL code for the ORDER BY part of a query
* @param string $sort_key is the key of $sort_by_sql for the selected sorting
* @param string $sort_dir is either a or d representing ASC and DESC
* @param string $sort_days specifies the maximum amount of days a post may be old
* @param array $ex_fid_ary specifies an array of forum ids which should not be searched
* @param string $post_visibility specifies which types of posts the user can view in which forums
* @param int $topic_id is set to 0 or a topic id, if it is not 0 then only posts in this topic should be searched
* @param array $author_ary an array of author ids if the author should be ignored during the search the array is empty
* @param string $author_name specifies the author match, when ANONYMOUS is also a search-match
* @param array &$id_ary passed by reference, to be filled with ids for the page specified by $start and $per_page, should be ordered
* @param int $start indicates the first index of the page
* @param int $per_page number of ids each page is supposed to contain
* @return boolean|int total number of results
*/
public function keyword_search($type, $fields, $terms, $sort_by_sql, $sort_key, $sort_dir, $sort_days, $ex_fid_ary, $post_visibility, $topic_id, $author_ary, $author_name, &$id_ary, &$start, $per_page)
{
// No keywords? No posts.
if (!strlen($this->search_query) && !sizeof($author_ary)) {
return false;
}
$id_ary = array();
$join_topic = $type != 'posts';
// Sorting
if ($type == 'topics') {
switch ($sort_key) {
case 'a':
$this->sphinx->SetGroupBy('topic_id', SPH_GROUPBY_ATTR, 'poster_id ' . ($sort_dir == 'a' ? 'ASC' : 'DESC'));
break;
case 'f':
$this->sphinx->SetGroupBy('topic_id', SPH_GROUPBY_ATTR, 'forum_id ' . ($sort_dir == 'a' ? 'ASC' : 'DESC'));
break;
case 'i':
case 's':
$this->sphinx->SetGroupBy('topic_id', SPH_GROUPBY_ATTR, 'post_subject ' . ($sort_dir == 'a' ? 'ASC' : 'DESC'));
break;
case 't':
default:
$this->sphinx->SetGroupBy('topic_id', SPH_GROUPBY_ATTR, 'topic_last_post_time ' . ($sort_dir == 'a' ? 'ASC' : 'DESC'));
break;
}
} else {
switch ($sort_key) {
case 'a':
$this->sphinx->SetSortMode($sort_dir == 'a' ? SPH_SORT_ATTR_ASC : SPH_SORT_ATTR_DESC, 'poster_id');
break;
case 'f':
$this->sphinx->SetSortMode($sort_dir == 'a' ? SPH_SORT_ATTR_ASC : SPH_SORT_ATTR_DESC, 'forum_id');
break;
case 'i':
case 's':
$this->sphinx->SetSortMode($sort_dir == 'a' ? SPH_SORT_ATTR_ASC : SPH_SORT_ATTR_DESC, 'post_subject');
break;
case 't':
default:
$this->sphinx->SetSortMode($sort_dir == 'a' ? SPH_SORT_ATTR_ASC : SPH_SORT_ATTR_DESC, 'post_time');
break;
}
}
// Most narrow filters first
if ($topic_id) {
$this->sphinx->SetFilter('topic_id', array($topic_id));
}
$search_query_prefix = '';
switch ($fields) {
case 'titleonly':
// Only search the title
if ($terms == 'all') {
$search_query_prefix = '@title ';
}
// Weight for the title
$this->sphinx->SetFieldWeights(array("title" => 5, "data" => 1));
// 1 is first_post, 0 is not first post
$this->sphinx->SetFilter('topic_first_post', array(1));
break;
case 'msgonly':
// Only search the body
if ($terms == 'all') {
$search_query_prefix = '@data ';
}
// Weight for the body
$this->sphinx->SetFieldWeights(array("title" => 1, "data" => 5));
break;
case 'firstpost':
// More relative weight for the title, also search the body
$this->sphinx->SetFieldWeights(array("title" => 5, "data" => 1));
// 1 is first_post, 0 is not first post
$this->sphinx->SetFilter('topic_first_post', array(1));
break;
default:
// More relative weight for the title, also search the body
$this->sphinx->SetFieldWeights(array("title" => 5, "data" => 1));
break;
}
if (sizeof($author_ary)) {
$this->sphinx->SetFilter('poster_id', $author_ary);
}
// As this is not simply possible at the moment, we limit the result to approved posts.
// This will make it impossible for moderators to search unapproved and softdeleted posts,
// but at least it will also cause the same for normal users.
$this->sphinx->SetFilter('post_visibility', array(ITEM_APPROVED));
if (sizeof($ex_fid_ary)) {
// All forums that a user is allowed to access
$fid_ary = array_unique(array_intersect(array_keys($this->auth->acl_getf('f_read', true)), array_keys($this->auth->acl_getf('f_search', true))));
// All forums that the user wants to and can search in
$search_forums = array_diff($fid_ary, $ex_fid_ary);
if (sizeof($search_forums)) {
$this->sphinx->SetFilter('forum_id', $search_forums);
}
}
$this->sphinx->SetFilter('deleted', array(0));
$this->sphinx->SetLimits($start, (int) $per_page, SPHINX_MAX_MATCHES);
$result = $this->sphinx->Query($search_query_prefix . str_replace('"', '"', $this->search_query), $this->indexes);
// Could be connection to localhost:9312 failed (errno=111,
// msg=Connection refused) during rotate, retry if so
$retries = SPHINX_CONNECT_RETRIES;
while (!$result && strpos($this->sphinx->GetLastError(), "errno=111,") !== false && $retries--) {
usleep(SPHINX_CONNECT_WAIT_TIME);
$result = $this->sphinx->Query($search_query_prefix . str_replace('"', '"', $this->search_query), $this->indexes);
}
if ($this->sphinx->GetLastError()) {
add_log('critical', 'LOG_SPHINX_ERROR', $this->sphinx->GetLastError());
if ($this->auth->acl_get('a_')) {
trigger_error($this->user->lang('SPHINX_SEARCH_FAILED', $this->sphinx->GetLastError()));
} else {
trigger_error($this->user->lang('SPHINX_SEARCH_FAILED_LOG'));
}
}
$result_count = $result['total_found'];
if ($result_count && $start >= $result_count) {
$start = floor(($result_count - 1) / $per_page) * $per_page;
$this->sphinx->SetLimits((int) $start, (int) $per_page, SPHINX_MAX_MATCHES);
$result = $this->sphinx->Query($search_query_prefix . str_replace('"', '"', $this->search_query), $this->indexes);
// Could be connection to localhost:9312 failed (errno=111,
// msg=Connection refused) during rotate, retry if so
$retries = SPHINX_CONNECT_RETRIES;
while (!$result && strpos($this->sphinx->GetLastError(), "errno=111,") !== false && $retries--) {
usleep(SPHINX_CONNECT_WAIT_TIME);
$result = $this->sphinx->Query($search_query_prefix . str_replace('"', '"', $this->search_query), $this->indexes);
}
}
$id_ary = array();
if (isset($result['matches'])) {
if ($type == 'posts') {
$id_ary = array_keys($result['matches']);
} else {
foreach ($result['matches'] as $key => $value) {
$id_ary[] = $value['attrs']['topic_id'];
}
}
} else {
return false;
}
$id_ary = array_slice($id_ary, 0, (int) $per_page);
return $result_count;
}
/**
* Получить ошибку при последнем обращении к поиску
*
* @return string
*/
public function GetLastError()
{
return $this->oSphinx->GetLastError();
}
/**
* Отправляет подготовленный запрос на сфинкс, и преобразует ответ в нужный вид.
*
* @param string $query поисковый запрос (в оригинальном виде)
* @param int $storeId ИД текущего магазина
* @param string $indexCode Код индекса по которому нужно провести поиск (mage_catalog_product ...)
* @param string $primaryKey Primary Key индекса (entity_id, category_id, post_id ...)
* @param array $attributes Масив атрибутов с весами
* @param int $offset Страница
*
* @return array масив ИД елементов, где ИД - ключ, релевантность значение
*/
protected function _query($query, $storeId, $index, $offset = 1)
{
$uid = Mage::helper('mstcore/debug')->start();
$indexCode = $index->getCode();
$primaryKey = $index->getPrimaryKey();
$attributes = $index->getAttributes();
$client = new SphinxClient();
$client->setMaxQueryTime(5000);
//5 seconds
$client->setLimits(($offset - 1) * self::PAGE_SIZE, self::PAGE_SIZE, $this->_config->getResultLimit());
$client->setSortMode(SPH_SORT_RELEVANCE);
$client->setMatchMode(SPH_MATCH_EXTENDED);
$client->setServer($this->_spxHost, $this->_spxPort);
$client->SetFieldWeights($attributes);
if ($storeId) {
$client->SetFilter('store_id', $storeId);
}
$sphinxQuery = $this->_buildQuery($query, $storeId);
if (!$sphinxQuery) {
return array();
}
$sphinxQuery = '@(' . implode(',', $index->getSearchableAttributes()) . ')' . $sphinxQuery;
$sphinxResult = $client->query($sphinxQuery, $indexCode);
if ($sphinxResult === false) {
Mage::throwException($client->GetLastError() . "\nQuery: " . $query);
} elseif ($sphinxResult['total'] > 0) {
$entityIds = array();
foreach ($sphinxResult['matches'] as $data) {
$entityIds[$data['attrs'][strtolower($primaryKey)]] = $data['weight'];
}
if ($sphinxResult['total'] > $offset * self::PAGE_SIZE && $offset * self::PAGE_SIZE < $this->_config->getResultLimit()) {
$newIds = $this->_query($query, $storeId, $index, $offset + 1);
foreach ($newIds as $key => $value) {
$entityIds[$key] = $value;
}
}
} else {
$entityIds = array();
}
$entityIds = $this->_normalize($entityIds);
Mage::helper('mstcore/debug')->end($uid, $entityIds);
return $entityIds;
}
/**
* Фильтр по аббривиатурам
*
* @Route("/ajax/abbr/{string}", name="ajax_abbr", defaults = {"string" = null})
*/
public function letterForAbbriviature(Request $request, $string = null)
{
if (!$string) {
$string = $request->query->get('string');
}
$type = $request->query->has('type') ? $request->query->get('type') : 'publication';
$order = $request->query->get('order', '') == 'relevant' ? 'relevant' : 'created';
//$order = $request->query->get('order', '') == 'created' ? 'created' : 'relevant';
$page = $request->query->has('page') ? $request->query->get('page') : $request->query->get('p', 1);
$page = intval($page);
# удаляем лишние символы и пробелы
$string = preg_replace('/[^a-zа-я\\s]/iu', ' ', $string);
$string = preg_replace('/\\s+/iu', ' ', $string);
$params = array('search' => $string);
$params['string'] = $string;
$params['type'] = $type;
$params['order'] = $order;
$params['page'] = $page;
$params['currentPage'] = $page;
if ($page < 1) {
throw $this->createNotFoundException('Incorrect page number: ' . $page);
}
# удаляем лишние символы из запроса
$string = preg_replace('/[^a-zа-я0-9-, :\\.\\(\\)]/iu', '', $string);
$string = preg_replace('/[,:\\.\\(\\)]/', ' ', $string);
# проверка, есть ли поисковое слово
if (mb_strlen(trim($string), 'utf-8') < 2) {
return $this->render('EvrikaMainBundle:Search:search_error.html.twig');
}
# разбиваем на слова, отсекаем окончания
if (!preg_match('/[A-Z]/', $string) && !preg_match('/[А-Я]/u', $string)) {
# берем основу слов, если нету собственных имен (заглавных)
$string = $this->get('evrika.lingua_stem')->stem_string($string);
}
# каждое слово дополняется с конца звездочкой (wildcard), разделитеть - ИЛИ
$words = explode(' ', $string);
# клиентский запрос
$s = new \SphinxClient();
$s->SetServer("localhost", 9312);
// должна быть запущена служба по порту: searchd --config /c/sphinx/shinx.cnf
//$s->SetRankingMode(SPH_RANK_PROXIMITY);
$s->SetMatchMode(SPH_MATCH_EXTENDED2);
// SPH_MATCH_ALL will match all words in the search term
$s->SetArrayResult(true);
$s->SetLimits(($page - 1) * self::SEARCH_PER_PAGE, self::SEARCH_PER_PAGE);
# получаем результаты поиска
if ($order == 'created') {
$s->setSortMode(SPH_SORT_EXTENDED, 'created DESC');
} else {
$s->SetFieldWeights(array('title' => 10, 'shortText' => 3, 'body' => 1));
$s->SetSortMode('SPH_SORT_EXTENDED', '@weight desc, mydate desc, @id asc');
}
# находим публикации по всем введенным словам
$query = implode('* &', $words) . '*';
$s_publications = $s->Query("@(title,shortText,body) {$query} & @type=publication", 'publication');
if (!isset($s_publications['matches'])) {
# если не нашли - находим по любому из слов
$query = implode('* |', $words) . '*';
$s_publications = $s->Query("@(title,shortText,body) {$query} & @type=publication", 'publication');
}
# находим события по всем введенным словам
$s_events = $s->Query("@(title,shortText,body) {$query} & @type=event", 'publication');
if (!isset($s_events['matches'])) {
# если не нашли - находим по любому из слов
$query = implode('* |', $words) . '*';
$s_events = $s->Query("@(title,shortText,body) {$query} & @type=event", 'publication');
}
# если поймали ошибку
$error = $s->GetLastError();
if (!empty($error)) {
throw $this->createNotFoundException($error);
}
# всего публикаций и событий
$params['totalPublications'] = intval($s_publications['total']);
$params['totalEvents'] = intval($s_events['total']);
# заполняем публикации/события из базы данных по полученным ID из запроса Sphinx
$publications = array();
$total = $type == 'event' ? $params['totalEvents'] : $params['totalPublications'];
# если материалы найдены
if ($total) {
$em = $this->getDoctrine()->getManager();
$highlight = array('publications' => array(), 'events' => array());
if ($type == 'event') {
foreach ($s_events['matches'] as $o) {
if ($publication = $em->getRepository('EvrikaMainBundle:Event')->findEnabledById($o['id'] - 1000000)) {
$publications[] = $publication;
}
}
} else {
foreach ($s_publications['matches'] as $o) {
if ($publication = $em->getRepository('EvrikaMainBundle:Publication')->findEnabledById($o['id'])) {
$publications[] = $publication;
}
}
}
# надо сделать подсветку частей с найденными словами
$words = explode(' ', $string);
$rep = array('$1<span class="yellow">$2</span>', '<span class="yellow">$0</span>');
for ($i = 0; $i < count($publications); $i++) {
$id = $publications[$i]->getId() . '';
$title = $publications[$i]->getTitle();
$shortText = $publications[$i]->getShortText();
$isEvent = $publications[$i]->isEvent();
foreach ($words as $word) {
if (!empty($word)) {
$pat = array('/([ ,>\\.\\-])(' . $word . '[a-zа-я]*)/iu', '/^' . $word . '[a-zа-я]*/iu');
$title = preg_replace($pat, $rep, $title);
if (!$isEvent) {
$shortText = preg_replace($pat, $rep, $shortText);
}
}
}
if ($isEvent) {
$highlight['events'][$id]['title'] = $title;
} else {
$highlight['publications'][$id]['title'] = $title;
$highlight['publications'][$id]['shortText'] = $shortText;
}
}
$params['highlight'] = $highlight;
}
# смотрим препараты
$emDrug = $this->getDoctrine()->getManager('drug');
list($products, $anyOfWord) = $emDrug->getRepository('VidalDrugBundle:Product')->findByQuery($string, false);
if (!empty($products)) {
$params['totalProducts'] = count($products);
if ($type == 'product') {
$productIds = array();
foreach ($products as $product) {
$productIds[] = $product['ProductID'];
}
$pagination = $this->get('knp_paginator')->paginate($products, $page, self::PRODUCTS_PER_PAGE);
$params['productsPagination'] = $pagination;
$params['anyOfWord'] = $anyOfWord;
$params['companies'] = $emDrug->getRepository('VidalDrugBundle:Company')->findByProducts($productIds);
$params['pictures'] = $emDrug->getRepository('VidalDrugBundle:Picture')->findByProductIds($productIds, date('Y'));
$params['infoPages'] = $emDrug->getRepository('VidalDrugBundle:InfoPage')->findByProducts($pagination);
}
}
# смотрим федеральные стандарты
$em = $this->getDoctrine()->getManager();
$standarts = $em->getRepository('EvrikaMainBundle:Standart')->findByQuery($string);
if (!empty($standarts)) {
$params['totalStandarts'] = count($standarts);
if ($type == 'standart') {
$params['standarts'] = $standarts;
}
}
$params['feedType'] = 'search';
$params['category'] = 'all';
$params['displayedPages'] = self::DISPLAYED_PAGES;
$params['bookmarks'] = $this->get('evrika.session_manager')->getBookmarks();
$params['type'] = $type;
$params['total'] = $total;
$params['numberOfPages'] = ceil($total / self::SEARCH_PER_PAGE);
$params['publications'] = $publications;
$return = array();
foreach ($publications as $item) {
$return[] = ['id' => $item->getId(), 'name' => $item->getTitle()];
}
return $this->render('EvrikaMainBundle:Abbreviation:ajax.json.twig', array('data' => $return));
}
private function doSearch($q, $page)
{
global $wgOut;
$mode = SPH_MATCH_ALL;
$index = 'suggested_titles';
$page_size = 20;
$limit = 1000;
$ranker = SPH_RANK_PROXIMITY_BM25;
$host = 'localhost';
$port = 9312;
$cl = new SphinxClient();
$cl->SetServer($host, $port);
//$cl->SetConnectTimeout(1);
$cl->SetSortMode(SPH_SORT_RELEVANCE);
$cl->SetArrayResult(true);
$cl->SetWeights(array('wst_title' => 5, 'wst_text' => 2));
$cl->SetMatchMode($mode);
$cl->SetRankingMode($ranker);
$cl->SetLimits(($page - 1) * $page_size, $page_size, $limit);
// don't search w/ leading "how to" if user added it
$q_prime = preg_replace('@^\\s*how\\s+to\\s+@i', '', $q);
$res = $cl->Query($q_prime, $index);
$error = $res === false ? $cl->GetLastError() : '';
$warning = $cl->GetLastWarning();
/*$spelling = $this->getSpellingInfo($q);
if ($spelling) {
$res['spelling'] = $spelling;
} else {
$res['spelling'] = '';
}*/
if (count($res['matches']) > 0) {
$titles = $this->getInfo($res['matches']);
$keys = array_keys($titles);
$excerpts = $cl->BuildExcerpts($titles, 'suggested_titles', $q);
foreach ($excerpts as $i => $excerpt) {
$excerpts[$keys[$i]] = $excerpt;
unset($excerpts[$i]);
}
foreach ($res['matches'] as $i => &$docinfo) {
$id = $docinfo['id'];
$docinfo['attrs']['excerpt'] = $excerpts[$id];
}
} else {
$error = wfMsg('search-keywords-not-found', $q);
}
// construct paging bar
$total = (int) ceil(1.0 * $res['total_found'] / $page_size);
$paging = array();
if ($page > 1) {
$paging[] = 'prev';
}
if ($page > 1) {
$paging[] = 1;
}
if ($page >= 5) {
$paging[] = '...';
}
if ($page >= 4) {
$paging[] = $page - 2;
}
if ($page >= 3) {
$paging[] = $page - 1;
}
$paging[] = $page;
if ($page < $total) {
$paging[] = $page + 1;
}
if ($page + 1 < $total) {
$paging[] = $page + 2;
}
if ($page + 2 < $total) {
$paging[] = '...';
}
if ($page < $total) {
$paging[] = 'next';
}
$vars = array('results' => $res, 'q' => $q, 'error' => $error, 'warning' => $warning, 'page' => $page, 'page_size' => $page_size, 'paging' => $paging);
return $vars;
}
public function __construct($rowsPerPage, $currentPage, $siteID, $wildCardString, $sortBy, $sortDirection)
{
$this->_db = DatabaseConnection::getInstance();
$this->_siteID = $siteID;
$this->_sortByFields = array('firstName', 'lastName', 'city', 'state', 'dateModifiedSort', 'dateCreatedSort', 'ownerSort');
if (ENABLE_SPHINX) {
/* Sphinx API likes to throw PHP errors *AND* use it's own error
* handling.
*/
assert_options(ASSERT_WARNING, 0);
$sphinx = new SphinxClient();
$sphinx->SetServer(SPHINX_HOST, SPHINX_PORT);
$sphinx->SetWeights(array(0, 100, 0, 0, 50));
$sphinx->SetMatchMode(SPH_MATCH_EXTENDED);
$sphinx->SetLimits(0, 1000);
$sphinx->SetSortMode(SPH_SORT_TIME_SEGMENTS, 'date_added');
// FIXME: This can be sped up a bit by actually grouping ranges of
// site IDs into their own index's. Maybe every 500 or so at
// least on the Hosted system.
$sphinx->SetFilter('site_id', array($this->_siteID));
/* Create the Sphinx query string. */
$wildCardString = DatabaseSearch::humanToSphinxBoolean($wildCardString);
/* Execute the Sphinx query. Sphinx can ask us to retry if its
* maxed out. Retry up to 5 times.
*/
$tries = 0;
do {
/* Wait for one second if this isn't out first attempt. */
if (++$tries > 1) {
sleep(1);
}
$results = $sphinx->Query($wildCardString, SPHINX_INDEX);
$errorMessage = $sphinx->GetLastError();
} while ($results === false && strpos($errorMessage, 'server maxed out, retry') !== false && $tries <= 5);
/* Throw a fatal error if Sphinx errors occurred. */
if ($results === false) {
$this->fatal('Sphinx Error: ' . ucfirst($errorMessage) . '.');
}
/* Throw a fatal error (for now) if Sphinx warnings occurred. */
$lastWarning = $sphinx->GetLastWarning();
if (!empty($lastWarning)) {
// FIXME: Just display a warning, and notify dev team.
$this->fatal('Sphinx Warning: ' . ucfirst($lastWarning) . '.');
}
/* Show warnings for assert()s again. */
assert_options(ASSERT_WARNING, 1);
if (empty($results['matches'])) {
$this->_WHERE = '0';
} else {
$attachmentIDs = implode(',', array_keys($results['matches']));
$this->_WHERE = 'attachment.attachment_id IN(' . $attachmentIDs . ')';
}
} else {
$tmp = DatabaseSearch::makeBooleanSQLWhere(DatabaseSearch::fulltextEncode($wildCardString), $this->_db, 'attachment.text');
$this->_WHERE = str_replace(") (", ") AND (", $tmp);
/*trace($tmp);
$matches=array();
preg_match_all('/"(?:\\\\.|[^\\\\"])*"|\S+/', $wildCardString, $matches);
$arrWhere=array();
$arrNew=array();
for($m=0;$m<count($matches[0]);$m++)
{
$match=$matches[0][$m];
if($match[0]=='"')
{
$arrNew[]=$match;
}
else
{
$tmp="";
for($n=$m;$n<count($matches[0]);$n++)
{
$match=$matches[0][$n];
if($match[0]=='"' || strtolower($match)=="and" || strtolower($match)=="or")
{
$m=$n-1;
break;
}
if($tmp==="")
{
$tmp=$match;
}
else
{
$tmp=$tmp." ".$match;
}
}
$tmp='"'.$tmp.'"';
$arrNew[]=$tmp;
}
}
foreach($arrNew as $match)
{
$arrWhere[] = DatabaseSearch::makeBooleanSQLWhere(
DatabaseSearch::fulltextEncode($match),
$this->_db,
'attachment.text'
);
}
$this->_WHERE=implode(" AND ",$arrWhere);*/
}
$sbase = new SearchBase();
$arrFilter = $sbase->buildFilter();
$filter = $arrFilter["where"];
$column = $arrFilter["extra_column"];
$join = $arrFilter["extra_join"];
$isSearchAttachment = isset($_REQUEST["bulk_resume"]) && $_REQUEST["bulk_resume"];
if ($isSearchAttachment) {
/* How many companies do we have? */
$sql = sprintf("SELECT\r\n count(*) AS count\r\n FROM\r\n attachment\r\n LEFT JOIN candidate\r\n ON attachment.data_item_id = candidate.candidate_id\r\n AND attachment.site_id = candidate.site_id\r\n LEFT JOIN user AS owner_user\r\n ON candidate.owner = owner_user.user_id\r\n %s\r\n WHERE\r\n resume = 1\r\n AND\r\n %s\r\n AND\r\n (attachment.data_item_type = %s OR attachment.data_item_type = %s)\r\n AND\r\n attachment.site_id = %s\r\n AND\r\n (ISNULL(candidate.is_admin_hidden) OR (candidate.is_admin_hidden = 0))\r\n AND\r\n (ISNULL(candidate.is_active) OR (candidate.is_active = 1))\r\n %s\r\n", $join, $this->_WHERE, DATA_ITEM_CANDIDATE, DATA_ITEM_BULKRESUME, $this->_siteID, $filter);
} else {
$sql = sprintf("SELECT\r\n count(*) AS count\r\n FROM\r\n attachment\r\n LEFT JOIN candidate\r\n ON attachment.data_item_id = candidate.candidate_id\r\n AND attachment.site_id = candidate.site_id\r\n LEFT JOIN user AS owner_user\r\n ON candidate.owner = owner_user.user_id\r\n %s\r\n WHERE\r\n resume = 1\r\n AND\r\n %s\r\n AND\r\n (attachment.data_item_type = %s)\r\n AND\r\n attachment.site_id = %s\r\n AND\r\n (ISNULL(candidate.is_admin_hidden) OR (candidate.is_admin_hidden = 0))\r\n AND\r\n (ISNULL(candidate.is_active) OR (candidate.is_active = 1))\r\n %s\r\n", $join, $this->_WHERE, DATA_ITEM_CANDIDATE, $this->_siteID, $filter);
}
$rs = $this->_db->getAssoc($sql);
/* Pass "Search By Resume"-specific parameters to Pager constructor. */
parent::__construct($rs['count'], $rowsPerPage, $currentPage);
}
$sphinx->SetServer(SPHINX_HOST, SPHINX_PORT);
$sphinx->SetWeights(array(0, 100, 0, 0, 50));
$sphinx->SetMatchMode(SPH_MATCH_BOOLEAN);
$sphinx->SetLimits(0, 10);
$sphinx->SetSortMode(SPH_SORT_TIME_SEGMENTS, 'date_added');
$sphinx->SetFilter('site_id', TEST_SITE_ID);
/* Execute the Sphinx query. Sphinx can ask us to retry if its
* maxed out. Retry up to 5 times.
*/
$tries = 0;
do {
/* Wait for one second if this isn't out first attempt. */
if (++$tries > 1) {
sleep(1);
}
$results = $sphinx->Query(TEST_QUERY, SPHINX_INDEX);
$errorMessage = $sphinx->GetLastError();
} while ($results === false && strpos($errorMessage, 'server maxed out, retry') !== false && $tries <= 5);
/* Throw a fatal error if Sphinx errors occurred. */
if ($results === false) {
fwrite($stderr, 'Sphinx Error: ' . ucfirst($errorMessage) . ".\n");
exit(1);
}
/* Throw a fatal error (for now) if Sphinx warnings occurred. */
$lastWarning = $sphinx->GetLastWarning();
if (!empty($lastWarning)) {
fwrite($stderr, 'Sphinx Warning: ' . ucfirst($lastWarning) . ".\n");
exit(1);
}
fwrite($stdout, "Sphinx appears to be working properly.\n");
exit(0);
/**
*
* sphinx instance
*/
private function sphinx_instance()
{
$sphinx = new SphinxClient();
$sphinx->setServer(Kohana::config('uap.sphinx_host'), Kohana::config('uap.sphinx_port'));
if (!$sphinx->open()) {
$this->send_response(400, NULL, '400111:cannot connect to sphinx daemon' . $sphinx->GetLastError());
}
return $sphinx;
}
