function add_search_words($mode, $post_id, $post_text, $post_title = '')
{
global $db, $config, $lang;
global $stopwords_array, $synonyms_array;
stopwords_synonyms_init();
$search_raw_words = array();
$search_raw_words['text'] = split_words(clean_words('post', $post_text, $stopwords_array, $synonyms_array));
$search_raw_words['title'] = split_words(clean_words('post', $post_title, $stopwords_array, $synonyms_array));
@set_time_limit(0);
$word = array();
$word_insert_sql = array();
while (list($word_in, $search_matches) = @each($search_raw_words)) {
$word_insert_sql[$word_in] = '';
if (!empty($search_matches)) {
for ($i = 0; $i < sizeof($search_matches); $i++) {
$search_matches[$i] = trim($search_matches[$i]);
if ($search_matches[$i] != '') {
$word[] = $search_matches[$i];
if (!strstr($word_insert_sql[$word_in], "'" . $search_matches[$i] . "'")) {
$word_insert_sql[$word_in] .= $word_insert_sql[$word_in] != "" ? ", '" . $search_matches[$i] . "'" : "'" . $search_matches[$i] . "'";
}
}
}
}
}
if (sizeof($word)) {
sort($word);
$prev_word = '';
$word_text_sql = '';
$temp_word = array();
for ($i = 0; $i < sizeof($word); $i++) {
if ($word[$i] != $prev_word) {
$temp_word[] = $word[$i];
$word_text_sql .= ($word_text_sql != '' ? ', ' : '') . "'" . $word[$i] . "'";
}
$prev_word = $word[$i];
}
$word = $temp_word;
$check_words = array();
$value_sql = '';
$match_word = array();
for ($i = 0; $i < sizeof($word); $i++) {
$new_match = true;
if (isset($check_words[$word[$i]])) {
$new_match = false;
}
if ($new_match) {
$value_sql .= ($value_sql != '' ? ', ' : '') . '(\'' . $word[$i] . '\', 0)';
}
}
if ($value_sql != '') {
$sql = "INSERT IGNORE INTO " . SEARCH_WORD_TABLE . " (word_text, word_common)\n\t\t\t\t\t\t\tVALUES {$value_sql}";
$db->sql_query($sql);
}
}
while (list($word_in, $match_sql) = @each($word_insert_sql)) {
$title_match = $word_in == 'title' ? 1 : 0;
if ($match_sql != '') {
$sql = "INSERT INTO " . SEARCH_MATCH_TABLE . " (post_id, word_id, title_match)\n\t\t\t\tSELECT {$post_id}, word_id, {$title_match}\n\t\t\t\t\tFROM " . SEARCH_WORD_TABLE . "\n\t\t\t\t\tWHERE word_text IN ({$match_sql})";
$db->sql_query($sql);
}
}
if ($mode == 'single') {
remove_common('single', 4 / 10, $word);
}
return;
}
$word_id_sql = '';
if ($row = $db->sql_fetchrow($result)) {
do {
$word_id_sql .= ($word_id_sql != '' ? ', ' : '') . $row['word_id'];
} while ($row = $db->sql_fetchrow($result));
$sql = "DELETE FROM " . SEARCH_WORD_TABLE . " \n\t\t\t\t\t\tWHERE word_id IN ({$word_id_sql})";
_sql($sql, $errored, $error_ary);
$sql = "DELETE FROM " . SEARCH_MATCH_TABLE . " \n\t\t\t\t\t\tWHERE word_id IN ({$word_id_sql})";
_sql($sql, $errored, $error_ary);
}
$db->sql_freeresult($result);
}
}
closedir($dir);
// Mark common words ...
remove_common('global', 4 / 10);
// remove superfluous polls ... grab polls with topics then delete polls
// not in that list
$sql = "SELECT v.vote_id \n\t\t\tFROM " . TOPICS_TABLE . " t, " . VOTE_DESC_TABLE . " v\n\t\t\tWHERE v.topic_id = t.topic_id";
$result = _sql($sql, $errored, $error_ary);
$vote_id_sql = '';
if ($row = $db->sql_fetchrow($result)) {
do {
$vote_id_sql .= ($vote_id_sql != '' ? ', ' : '') . $row['vote_id'];
} while ($row = $db->sql_fetchrow($result));
$sql = "DELETE FROM " . VOTE_DESC_TABLE . " \n\t\t\t\tWHERE vote_id NOT IN ({$vote_id_sql})";
_sql($sql, $errored, $error_ary);
$sql = "DELETE FROM " . VOTE_RESULTS_TABLE . " \n\t\t\t\tWHERE vote_id NOT IN ({$vote_id_sql})";
_sql($sql, $errored, $error_ary);
$sql = "DELETE FROM " . VOTE_USERS_TABLE . " \n\t\t\t\tWHERE vote_id NOT IN ({$vote_id_sql})";
_sql($sql, $errored, $error_ary);
function add_search_words($mode, $post_id, $post_text, $post_title = '')
{
global $db, $phpbb_root_path, $board_config, $lang;
$stopword_array = @file($phpbb_root_path . 'language/lang_' . $board_config['default_lang'] . "/search_stopwords.txt");
$synonym_array = @file($phpbb_root_path . 'language/lang_' . $board_config['default_lang'] . "/search_synonyms.txt");
$search_raw_words = array();
$search_raw_words['text'] = split_words(clean_words('post', $post_text, $stopword_array, $synonym_array));
$search_raw_words['title'] = split_words(clean_words('post', $post_title, $stopword_array, $synonym_array));
@set_time_limit(0);
$word = array();
$word_insert_sql = array();
while (list($word_in, $search_matches) = @each($search_raw_words)) {
$word_insert_sql[$word_in] = '';
if (!empty($search_matches)) {
for ($i = 0; $i < count($search_matches); $i++) {
$search_matches[$i] = trim($search_matches[$i]);
if ($search_matches[$i] != '') {
$word[] = $search_matches[$i];
if (!strstr($word_insert_sql[$word_in], "'" . $search_matches[$i] . "'")) {
$word_insert_sql[$word_in] .= $word_insert_sql[$word_in] != "" ? ", '" . $search_matches[$i] . "'" : "'" . $search_matches[$i] . "'";
}
}
}
}
}
if (count($word)) {
sort($word);
$prev_word = '';
$word_text_sql = '';
$temp_word = array();
for ($i = 0; $i < count($word); $i++) {
if ($word[$i] != $prev_word) {
$temp_word[] = $word[$i];
$word_text_sql .= ($word_text_sql != '' ? ', ' : '') . "'" . $word[$i] . "'";
}
$prev_word = $word[$i];
}
$word = $temp_word;
$check_words = array();
switch (SQL_LAYER) {
case 'postgresql':
case 'msaccess':
case 'mssql-odbc':
case 'oracle':
case 'db2':
$sql = "SELECT word_id, word_text\r\n FROM " . SEARCH_WORD_TABLE . "\r\n WHERE word_text IN ({$word_text_sql})";
if (!($result = $db->sql_query($sql))) {
message_die(GENERAL_ERROR, 'Could not select words', '', __LINE__, __FILE__, $sql);
}
while ($row = $db->sql_fetchrow($result)) {
$check_words[$row['word_text']] = $row['word_id'];
}
break;
}
$value_sql = '';
$match_word = array();
for ($i = 0; $i < count($word); $i++) {
$new_match = true;
if (isset($check_words[$word[$i]])) {
$new_match = false;
}
if ($new_match) {
switch (SQL_LAYER) {
case 'mysql':
case 'mysql4':
$value_sql .= ($value_sql != '' ? ', ' : '') . '(\'' . $word[$i] . '\', 0)';
break;
case 'mssql':
case 'mssql-odbc':
$value_sql .= ($value_sql != '' ? ' UNION ALL ' : '') . "SELECT '" . $word[$i] . "', 0";
break;
default:
$sql = "INSERT INTO " . SEARCH_WORD_TABLE . " (word_text, word_common)\r\n VALUES ('" . $word[$i] . "', '0')";
if (!$db->sql_query($sql)) {
message_die(GENERAL_ERROR, 'Could not insert new word', '', __LINE__, __FILE__, $sql);
}
break;
}
}
}
if ($value_sql != '') {
switch (SQL_LAYER) {
case 'mysql':
case 'mysql4':
$sql = "INSERT IGNORE INTO " . SEARCH_WORD_TABLE . " (word_text, word_common)\r\n VALUES {$value_sql}";
break;
case 'mssql':
case 'mssql-odbc':
$sql = "INSERT INTO " . SEARCH_WORD_TABLE . " (word_text, word_common)\r\n {$value_sql}";
break;
}
if (!$db->sql_query($sql)) {
message_die(GENERAL_ERROR, 'Could not insert new word', '', __LINE__, __FILE__, $sql);
}
}
}
while (list($word_in, $match_sql) = @each($word_insert_sql)) {
$title_match = $word_in == 'title' ? 1 : 0;
if ($match_sql != '') {
$sql = "INSERT INTO " . SEARCH_MATCH_TABLE . " (post_id, word_id, title_match)\r\n SELECT {$post_id}, word_id, {$title_match}\r\n FROM " . SEARCH_WORD_TABLE . "\r\n WHERE word_text IN ({$match_sql})";
if (!$db->sql_query($sql)) {
message_die(GENERAL_ERROR, 'Could not insert new word matches', '', __LINE__, __FILE__, $sql);
}
}
}
if ($mode == 'single') {
remove_common('single', 4 / 10, $word);
}
return;
}
$inc = 0;
if ($row = $db->sql_fetchrow($posts_result)) {
do {
add_search_words($row['post_id'], $row['post_text'], $row['post_subject']);
$inc++;
if ($inc == $per_pct) {
print ".";
flush();
$inc = 0;
}
} while ($row = $db->sql_fetchrow($posts_result));
}
$db->sql_freeresult($posts_result);
// Remove common words after the first 2 batches and after every 4th batch after that.
if ($batchcount % 4 == 3) {
remove_common('global', 0.4);
}
print " <span class=\"ok\"><b>OK</b></span><br />\n";
}
end_step('update_topics');
case 'update_topics':
$sql = "SELECT MAX(topic_id) AS max_topic \r\n\t\t\t\tFROM " . TOPICS_TABLE;
$result = query($sql, "Couldn't get max topic id");
$row = $db->sql_fetchrow($result);
$maxid = $row['max_topic'];
lock_tables(1, array(TOPICS_TABLE, POSTS_TABLE));
$batchsize = 1000;
for ($i = 0; $i <= $maxid; $i += $batchsize) {
$batchstart = $i + 1;
$batchend = $i + $batchsize;
print " * Setting topic first post_id ( {$batchstart} to {$batchend} ) :: ";
}
if (!is_null($cache_word_id) && ($last_post_id != $result_array[0][$i] || $last_word_id != $cache_word_id || $last_title_match != $result_array[1][$i])) {
$last_post_id = $result_array[0][$i];
$last_word_id = $cache_word_id;
$last_title_match = $result_array[1][$i];
$sql = "INSERT INTO " . SEARCH_MATCH_TABLE . " (post_id, word_id, title_match) VALUES ({$last_post_id}, {$last_word_id}, {$last_title_match})";
if (!$db->sql_query($sql)) {
include './page_header_admin.' . $phpEx;
throw_error("Couldn't insert into search match!", __LINE__, __FILE__, $sql);
}
}
unset($result_array[0][$i]);
unset($result_array[1][$i]);
unset($result_array[2][$i]);
}
remove_common('single', 4 / 10, $word_array);
break;
}
// All posts are indexed for this turn - update Config-Data
update_config('dbmtnc_rebuild_pos', $last_post);
// OK, all actions are done - send headers
$template->assign_vars(array('META' => '<meta http-equiv="refresh" content="1;url=' . append_sid("admin_db_maintenance.{$phpEx}?mode=perform&function=perform_rebuild&db_state={$db_state}") . '">'));
include './page_header_admin.' . $phpEx;
ob_end_flush();
$db->sql_freeresult($result);
// Get Statistics
$posts_total = 0;
$sql = "SELECT Count(*) AS posts_total\n\t\t\t\t\tFROM " . POSTS_TEXT_TABLE . "\n\t\t\t\t\tWHERE post_id <= " . intval($board_config['dbmtnc_rebuild_end']);
if ($result = $db->sql_query($sql)) {
if ($row = $db->sql_fetchrow($result)) {
$posts_total = $row['posts_total'];
function add_search_words($mode, $post_id, $post_text, $post_title = '')
{
$stopword_array = @file("core/lib/phpbb/search_stopwords.txt");
$synonym_array = @file("core/lib/phpbb/search_synonyms.txt");
$search_raw_words = array();
$search_raw_words['text'] = split_words(clean_words('post', $post_text, $stopword_array, $synonym_array));
$search_raw_words['title'] = split_words(clean_words('post', $post_title, $stopword_array, $synonym_array));
@set_time_limit(0);
$word = array();
$word_insert_sql = array();
while (list($word_in, $search_matches) = @each($search_raw_words)) {
$word_insert_sql[$word_in] = '';
if (!empty($search_matches)) {
for ($i = 0; $i < count($search_matches); $i++) {
$search_matches[$i] = trim($search_matches[$i]);
if ($search_matches[$i] != '') {
$word[] = $search_matches[$i];
if (!strstr($word_insert_sql[$word_in], "'" . $search_matches[$i] . "'")) {
$word_insert_sql[$word_in] .= $word_insert_sql[$word_in] != "" ? ", '" . $search_matches[$i] . "'" : "'" . $search_matches[$i] . "'";
}
}
}
}
}
if (count($word)) {
sort($word);
$prev_word = '';
$word_text_sql = '';
$temp_word = array();
for ($i = 0; $i < count($word); $i++) {
if ($word[$i] != $prev_word) {
$temp_word[] = $word[$i];
$word_text_sql .= ($word_text_sql != '' ? ', ' : '') . "'" . $word[$i] . "'";
}
$prev_word = $word[$i];
}
$word = $temp_word;
$check_words = array();
$value_sql = '';
$match_word = array();
for ($i = 0; $i < count($word); $i++) {
$new_match = true;
if (isset($check_words[$word[$i]])) {
$new_match = false;
}
if ($new_match) {
$value_sql .= ($value_sql != '' ? ', ' : '') . '(\'' . $word[$i] . '\', 0)';
}
}
if ($value_sql != '') {
$sql = "INSERT IGNORE INTO `z_phpbb_search_wordlist` (word_text, word_common) \n VALUES {$value_sql}";
mysql_query($sql);
}
}
while (list($word_in, $match_sql) = @each($word_insert_sql)) {
$title_match = $word_in == 'title' ? 1 : 0;
if ($match_sql != '') {
$sql = "INSERT INTO z_phpbb_search_wordmatch (post_id, word_id, title_match) \n SELECT {$post_id}, word_id, {$title_match} \n FROM z_phpbb_search_wordlist \n WHERE word_text IN ({$match_sql})";
mysql_query($sql);
}
}
if ($mode == 'single') {
remove_common('single', 4 / 10, $word);
}
return;
}
function add_search_words($mode, $post_id, $post_text, $post_title = '', $topic_id = '')
{
global $ft_cfg, $lang;
$stopword_array = @file(FT_ROOT . 'language/lang_' . $ft_cfg['default_lang'] . "/search_stopwords.txt");
$synonym_array = @file(FT_ROOT . 'language/lang_' . $ft_cfg['default_lang'] . "/search_synonyms.txt");
$search_raw_words = array();
$search_raw_words['text'] = split_words(clean_words('post', $post_text, $stopword_array, $synonym_array));
$search_raw_words['title'] = split_words(clean_words('post', $post_title, $stopword_array, $synonym_array));
@set_time_limit(0);
$word = array();
$word_insert_sql = array();
while (list($word_in, $search_matches) = @each($search_raw_words)) {
$word_insert_sql[$word_in] = '';
if (!empty($search_matches)) {
for ($i = 0, $cnt = count($search_matches); $i < $cnt; $i++) {
$search_matches[$i] = @$search_matches[$i] ? trim($search_matches[$i]) : '';
if ($search_matches[$i] != '') {
$word[] = $search_matches[$i];
if (!strstr($word_insert_sql[$word_in], "'" . $search_matches[$i] . "'")) {
$word_insert_sql[$word_in] .= $word_insert_sql[$word_in] != "" ? ", '" . $search_matches[$i] . "'" : "'" . $search_matches[$i] . "'";
}
}
}
}
}
if (count($word)) {
sort($word);
$prev_word = '';
$word_text_sql = '';
$temp_word = array();
for ($i = 0; $i < count($word); $i++) {
if ($word[$i] != $prev_word) {
$temp_word[] = $word[$i];
$word_text_sql .= ($word_text_sql != '' ? ', ' : '') . "'" . $word[$i] . "'";
}
$prev_word = $word[$i];
}
$word = $temp_word;
$check_words = array();
$value_sql = '';
$match_word = array();
for ($i = 0; $i < count($word); $i++) {
$new_match = true;
if (isset($check_words[$word[$i]])) {
$new_match = false;
}
if ($new_match) {
$sql = "INSERT INTO " . SEARCH_WORD_TABLE . " (word_text, word_common)\n\t\t\t\t\tVALUES ('" . $word[$i] . "', 0)";
if (!DB()->sql_query($sql)) {
message_die(GENERAL_ERROR, 'Could not insert new word', '', __LINE__, __FILE__, $sql);
}
}
}
if ($value_sql != '') {
$sql = "INSERT IGNORE INTO " . SEARCH_WORD_TABLE . " (word_text, word_common)\n\t\t\t\tVALUES {$value_sql}";
if (!DB()->sql_query($sql)) {
message_die(GENERAL_ERROR, 'Could not insert new word', '', __LINE__, __FILE__, $sql);
}
}
}
while (list($word_in, $match_sql) = @each($word_insert_sql)) {
$title_match = $word_in == 'title' ? 1 : 0;
if ($match_sql != '') {
$sql = "INSERT INTO " . SEARCH_MATCH_TABLE . " (post_id, word_id, title_match)\n\t\t\t\tSELECT {$post_id}, word_id, {$title_match}\n\t\t\t\t\tFROM " . SEARCH_WORD_TABLE . "\n\t\t\t\t\tWHERE word_text IN ({$match_sql})";
if (!DB()->sql_query($sql)) {
message_die(GENERAL_ERROR, 'Could not insert new word matches', '', __LINE__, __FILE__, $sql);
}
}
}
if ($mode == 'single') {
remove_common('single', 4 / 10, $word);
}
return;
}
