/**
* Gets upload Limits
*
* @return array with limits
* @since 3.0
*/
static function getPHPuploadLimit()
{
$post_max = flexicontent_upload::parseByteLimit(ini_get('post_max_size'));
$upload_max = flexicontent_upload::parseByteLimit(ini_get('upload_max_filesize'));
if ($upload_max < $post_max) {
$limit = array('value' => $upload_max, 'name' => 'upload_max_filesize');
} else {
$limit = array('value' => $post_max, 'name' => 'post_max_size');
}
// Sucosin limitation
if (extension_loaded('suhosin')) {
$post_max = flexicontent_upload::parseByteLimit(ini_get('suhosin.post.max_value_length'));
if ($post_max < $limit['value']) {
$limit = array('value' => $post_max, 'name' => 'suhosin.post.max_value_length');
}
}
return $limit;
}
function index()
{
$start_microtime = microtime(true);
$session = JFactory::getSession();
$db = JFactory::getDBO();
// Test indexing with limited memory
//ini_set("memory_limit", "20M");
@ob_end_clean();
$search_prefix = JComponentHelper::getParams('com_flexicontent')->get('add_search_prefix') ? 'vvv' : '';
// SEARCH WORD Prefix
$indexer = JRequest::getVar('indexer', 'advanced');
$rebuildmode = JRequest::getVar('rebuildmode', '');
$items_per_call = JRequest::getVar('items_per_call', 20);
// Number of item to index per HTTP request
$itemcnt = JRequest::getVar('itemcnt', 0);
// Counter of items indexed so far, this is given via HTTP request
// TAKE CARE: this code depends on countrows() to set session variables
// Retrieve fields, that are assigned as (advanced/basic) searchable/filterable
if ($rebuildmode == 'quick' && $indexer == 'advanced') {
$nse_fields = $session->get($indexer . '_nse_fields', array(), 'flexicontent');
$nsp_fields = $session->get($indexer . '_nsp_fields', array(), 'flexicontent');
$fields = $session->get($indexer . '_fields', array(), 'flexicontent');
//echo 'fail|'; print_r(array_keys($fields)); exit;
// Get the field ids of the fields removed from searching
$del_fieldids = array_unique(array_merge(array_keys($nse_fields), array_keys($nsp_fields), array_keys($fields)));
} else {
$fields = $session->get($indexer . '_fields', array(), 'flexicontent');
//echo 'fail|'; print_r(array_keys($fields)); exit;
}
// Get the field ids of the searchable fields
$fieldids = array_keys($fields);
// Get fields that will have atomic search tables, (current for advanced index only)
if ($indexer == 'advanced') {
$filterables = FlexicontentFields::getSearchFields('id', $indexer, null, null, $_load_params = false, 0, $search_type = 'filter');
$filterables = array_keys($filterables);
$filterables = array_flip($filterables);
} else {
$filterables = array();
}
// Get items ids that have value for any of the searchable fields, but use session to avoid recalculation
$itemids = $session->get($indexer . '_items_to_index', array(), 'flexicontent');
$_fields = array();
foreach ($fields as $field_id => $field) {
// Clone field to avoid problems
$_fields[$field_id] = clone $field;
// Create field parameters if not already created
if (empty($_fields[$field_id]->parameters)) {
$_fields[$field_id]->parameters = new JRegistry($_fields[$field_id]->attribs);
}
}
$fields = $_fields;
// Get query size limit
$query = "SHOW VARIABLES LIKE 'max_allowed_packet'";
$db->setQuery($query);
$_dbvariable = $db->loadObject();
$max_allowed_packet = flexicontent_upload::parseByteLimit(@$_dbvariable->Value);
$max_allowed_packet = $max_allowed_packet ? $max_allowed_packet : 256 * 1024;
$query_lim = (int) (3 * $max_allowed_packet / 4);
//echo 'fail|'.$query_lim; exit;
// Get script max
$max_execution_time = ini_get("max_execution_time");
//echo 'fail|'.$max_execution_time; exit;
$query_count = 0;
$max_items_per_query = 100;
$max_items_per_query = $max_items_per_query > $items_per_call ? $items_per_call : $max_items_per_query;
$cnt = $itemcnt;
while ($cnt < count($itemids) && $cnt < $itemcnt + $items_per_call) {
$query_itemids = array_slice($itemids, $cnt, $max_items_per_query);
$cnt += $max_items_per_query;
// Item is not needed, later and only if field uses item replacements then it will be loaded
$item = null;
// Items language is needed to do (if needed) special per language handling
$lang_query = "SELECT id, language" . " FROM #__content AS i " . " WHERE id IN (" . implode(', ', $query_itemids) . ")";
$db->setQuery($lang_query);
$items_data = $db->loadObjectList('id');
if ($indexer == 'basic') {
$searchindex = array();
// Add all query itemids to searchindex array so that it will be cleared even if zero fields are indexed
foreach ($query_itemids as $query_itemid) {
$searchindex[$query_itemid] = array();
}
} else {
// This will hold the SQL inserting new advanced search records for multiple item/values
$ai_query_vals = array();
$ai_query_vals_f = array();
// Current for advanced index only
}
// For current item: Loop though all searchable fields according to their type
foreach ($fieldids as $fieldid) {
// Must SHALLOW clone because we will be setting some properties , e.g. 'ai_query_vals', that we do not
$field = clone $fields[$fieldid];
// Indicate multiple items per query
$field->item_id = 0;
$field->query_itemids = $query_itemids;
$field->items_data = $items_data;
// Includes item langyage, which may be used for special per language handling
// Indicate that the indexing fuction should retrieve the values
$values = null;
// Add values to advanced search index
$fieldname = $field->iscore ? 'core' : $field->field_type;
if ($indexer == 'advanced') {
FLEXIUtilities::call_FC_Field_Func($fieldname, 'onIndexAdvSearch', array(&$field, &$values, &$item));
//print_r($field->ai_query_vals);
if (isset($field->ai_query_vals)) {
foreach ($field->ai_query_vals as $query_val) {
$ai_query_vals[] = $query_val;
}
if (isset($filterables[$field->id])) {
// Current for advanced index only
foreach ($field->ai_query_vals as $query_val) {
$ai_query_vals_f[$field->id][] = $query_val;
}
}
}
//else echo "Not set for : ". $field->name;
} else {
if ($indexer == 'basic') {
FLEXIUtilities::call_FC_Field_Func($fieldname, 'onIndexSearch', array(&$field, &$values, &$item));
foreach ($query_itemids as $query_itemid) {
if (@$field->search[$query_itemid]) {
$searchindex[$query_itemid][] = $field->search[$query_itemid];
}
}
}
}
}
// Create query that will update/insert data into the DB
unset($queries);
// make sure it is not set above
$queries = array();
if ($indexer == 'basic') {
if (count($searchindex)) {
// check for zero search index records
$query_vals = '';
$query_ids = array();
// Start new query
foreach ($searchindex as $query_itemid => $search_text) {
if (strlen($query_vals) > $query_lim) {
$query = "UPDATE #__flexicontent_items_ext SET search_index = CASE item_id " . $query_vals . " END " . " WHERE item_id IN (" . implode(',', $query_ids) . ")";
$queries[] = $query;
$query_vals = '';
$query_ids = array();
// Start new query
}
$query_ids[] = $query_itemid;
$_search_text = implode(' | ', $search_text);
if ($search_prefix && $_search_text) {
$_search_text = preg_replace('/(\\b[^\\s,\\.]+\\b)/u', $search_prefix . '$0', trim($_search_text));
}
$query_vals .= " WHEN {$query_itemid} THEN " . $db->Quote($_search_text);
}
if (count($query_ids)) {
$query = "UPDATE #__flexicontent_items_ext SET search_index = CASE item_id " . $query_vals . " END " . " WHERE item_id IN (" . implode(',', $query_ids) . ")";
$queries[] = $query;
}
}
} else {
if (count($ai_query_vals)) {
// check for zero search index records
$query_vals = '';
// Start new query
foreach ($ai_query_vals as &$query_value) {
$query_vals .= ($query_vals ? ',' : '') . $query_value;
if (strlen($query_vals) > $query_lim) {
$queries[] = "INSERT INTO #__flexicontent_advsearch_index " . " (field_id,item_id,extraid,search_index,value_id) VALUES " . $query_vals;
$query_vals = '';
// Start new query
}
}
unset($query_value);
if (strlen($query_vals)) {
$queries[] = "INSERT INTO #__flexicontent_advsearch_index " . " (field_id,item_id,extraid,search_index,value_id) VALUES " . $query_vals;
}
}
foreach ($ai_query_vals_f as $_field_id => $_query_vals) {
$query_vals = '';
// Start new query
foreach ($_query_vals as &$query_value) {
$query_vals .= ($query_vals ? ',' : '') . $query_value;
if (strlen($query_vals) > $query_lim) {
$queries[] = "INSERT INTO #__flexicontent_advsearch_index_field_" . $_field_id . " (field_id,item_id,extraid,search_index,value_id) VALUES " . $query_vals;
$query_vals = '';
// Start new query
}
}
if (strlen($query_vals)) {
$queries[] = "INSERT INTO #__flexicontent_advsearch_index_field_" . $_field_id . " (field_id,item_id,extraid,search_index,value_id) VALUES " . $query_vals;
$query_vals = '';
// Start new query
}
}
}
foreach ($queries as $query) {
$db->setQuery($query);
try {
$db->execute();
} catch (RuntimeException $e) {
echo "fail|" . $e->getMessage();
exit;
}
}
$query_count += count($queries);
$elapsed_microseconds = round(1000000 * 10 * (microtime(true) - $start_microtime)) / 10;
$elapsed_seconds = $elapsed_microseconds / 1000000.0;
if ($elapsed_seconds > $max_execution_time / 3 || $elapsed_seconds > 5) {
break;
}
}
// Check if items have finished, otherwise continue with -next- group of item ids
if ($cnt >= count($itemids)) {
// Reset dirty SEARCH properties of published fields to be: normal ON/OFF
$set_clause = ' SET' . ($indexer == 'basic' ? ' issearch = CASE issearch WHEN 2 THEN 1 WHEN -1 THEN 0 ELSE issearch END' : ' isadvsearch = CASE isadvsearch WHEN 2 THEN 1 WHEN -1 THEN 0 ELSE isadvsearch END,' . ' isadvfilter = CASE isadvfilter WHEN 2 THEN 1 WHEN -1 THEN 0 ELSE isadvfilter END');
$query = 'UPDATE #__flexicontent_fields' . $set_clause . " WHERE published=1";
$db->setQuery($query);
$db->execute();
// Force SEARCH properties of unpublished fields to be: normal OFF
if ($indexer == 'basic') {
$query = 'UPDATE #__flexicontent_fields SET issearch = 0 WHERE published=0';
$db->setQuery($query);
$db->execute();
} else {
$query = 'UPDATE #__flexicontent_fields SET isadvsearch = 0, isadvfilter = 0 WHERE published=0';
$db->setQuery($query);
$db->execute();
}
}
if (!count($fieldids)) {
echo 'fail|Index was only cleaned-up, <br/>since no <b>fields</b> were marked as: ' . '<br> -- ' . ($indexer == 'basic' ? 'Text Searchable (CONTENT LISTS)' : 'Text Searchable OR filterable (SEARCH VIEW)');
exit;
}
if (!count($itemids)) {
echo 'fail|Index was only cleaned-up, <br/>since no <b>items</b> were found to have value for fields marked as: ' . '<br> -- ' . ($indexer == 'basic' ? 'Text Searchable (CONTENT LISTS)' : 'Text Searchable OR filterable (SEARCH VIEW)');
exit;
}
$elapsed_microseconds = round(1000000 * 10 * (microtime(true) - $start_microtime)) / 10;
if ($session->has($indexer . '_total_runtime', 'flexicontent')) {
$_total_runtime = $session->get($indexer . '_total_runtime', 0, 'flexicontent');
} else {
$_total_runtime = 0;
}
$_total_runtime += $elapsed_microseconds;
$session->set($indexer . '_total_runtime', $_total_runtime, 'flexicontent');
if ($session->has($indexer . '_total_queries', 'flexicontent')) {
$_total_queries = $session->get($indexer . '_total_queries', 0, 'flexicontent');
} else {
$_total_queries = 0;
}
$_total_queries += $query_count;
$session->set($indexer . '_total_queries', $_total_queries, 'flexicontent');
echo sprintf($cnt . ' | Server execution time: %.2f secs ', $_total_runtime / 1000000) . ' | Total DB updates: ' . $_total_queries;
exit;
}
/**
* Method to add flexi extended datas to standard content
*
* @params object the unassociated items rows
* @params boolean add the records from the items_ext table
* @return boolean
* @since 1.5
*/
function bindExtData($rows)
{
if (!$rows || !count($rows)) {
return;
}
$app = JFactory::getApplication();
$jinput = $app->input;
$search_prefix = $this->cparams->get('add_search_prefix') ? 'vvv' : '';
// SEARCH WORD Prefix
$typeid = $jinput->get('typeid', 1, 'int');
$default_cat = $jinput->get('default_cat', 0, 'int');
$default_lang = flexicontent_html::getSiteDefaultLang();
// Get invalid cats, to avoid using them during binding, this is only done once
$session = JFactory::getSession();
$badcats_fixed = $session->get('badcats', null, 'flexicontent');
if ($badcats_fixed === null) {
// Correct non-existent main category in content table
$query = 'UPDATE #__content as c ' . ' LEFT JOIN #__categories as cat ON c.catid=cat.id' . ' SET c.catid=' . $default_cat . ' WHERE cat.id IS NULL';
$this->_db->setQuery($query);
$this->_db->execute();
$session->set('badcats_fixed', 1, 'flexicontent');
}
// Calculate item data to be used for current bind STEP
$catrel = array();
foreach ($rows as $row) {
$row_catid = (int) $row->catid;
$catrel[] = '(' . $row_catid . ', ' . (int) $row->id . ')';
// append the text property to the object
if (JString::strlen($row->fulltext) > 1) {
$row->text_stripped = $row->introtext . '<hr id="system-readmore" />' . $row->fulltext;
} else {
$row->text_stripped = flexicontent_html::striptagsandcut($row->introtext);
}
}
// Insert main category-item relation via single query
$catrel = implode(', ', $catrel);
$query = "INSERT INTO #__flexicontent_cats_item_relations (`catid`, `itemid`) " . " VALUES " . $catrel . " ON DUPLICATE KEY UPDATE ordering=ordering";
$this->_db->setQuery($query);
$this->_db->execute();
$query = "SHOW VARIABLES LIKE 'max_allowed_packet'";
$this->_db->setQuery($query);
$_dbvariable = $this->_db->loadObject();
$max_allowed_packet = flexicontent_upload::parseByteLimit(@$_dbvariable->Value);
$max_allowed_packet = $max_allowed_packet ? $max_allowed_packet : 256 * 1024;
$query_lim = (int) (3 * $max_allowed_packet / 4);
// Insert items_ext datas,
// NOTE: we will not use a single query for creating multiple records, instead we will create only e.g. 100 at once,
// because of the column search_index which can be quite long
$itemext = array();
$id_arr = array();
$row_count = count($rows);
$n = 0;
$i = 0;
$query_len = 0;
foreach ($rows as $row) {
$ilang = $row->language ? $row->language : $default_lang;
if ($search_prefix) {
$_search_index = preg_replace('/(\\b[^\\s,\\.]+\\b)/u', $search_prefix . '$0', $row->title . ' | ' . $row->text_stripped);
} else {
$_search_index = $row->title . ' | ' . $row->text_stripped;
}
$itemext[$i] = '(' . (int) $row->id . ', ' . $typeid . ', ' . $this->_db->Quote($ilang) . ', ' . $this->_db->Quote($_search_index) . ', 0)';
$id_arr[$i] = (int) $row->id;
$query_len += strlen($itemext[$i]) + 2;
// Sum of query length so far
$n++;
$i++;
if ($n % 101 == 0 || $n == $row_count || $query_len > $query_lim) {
$itemext_list = implode(', ', $itemext);
$query = "INSERT INTO #__flexicontent_items_ext (`item_id`, `type_id`, `language`, `search_index`, `lang_parent_id`)" . " VALUES " . $itemext_list . " ON DUPLICATE KEY UPDATE type_id=VALUES(type_id), language=VALUES(language), search_index=VALUES(search_index)";
$this->_db->setQuery($query);
$this->_db->execute();
// reset the item array
$itemext = array();
$query = "UPDATE #__flexicontent_items_tmp" . " SET type_id=" . $typeid . " WHERE id IN(" . implode(',', $id_arr) . ")";
$this->_db->setQuery($query);
$this->_db->execute();
// reset the item id array
$id_arr = array();
$i = 0;
// reset sub-counter, and query length
$query_len = 0;
}
}
// Update temporary item data
$this->updateItemCountingData($rows);
}
function index()
{
$start_microtime = microtime(true);
$session = JFactory::getSession();
$db = JFactory::getDBO();
@ob_end_clean();
$indexer = JRequest::getVar('indexer', 'fileman_default');
$rebuildmode = JRequest::getVar('rebuildmode', '');
$items_per_call = JRequest::getVar('items_per_call', 20);
// Number of item to index per HTTP request
$itemcnt = JRequest::getVar('itemcnt', 0);
// Counter of items indexed so far, this is given via HTTP request
// Actions according to rebuildmode
if ($indexer != 'fileman_default') {
die("'rebuildmode': '" . $rebuildmode . "'. not supported");
}
// Get items ids that have value for any of the searchable fields, but use session to avoid recalculation
$itemids = $session->get($indexer . '_items_to_index', array(), 'flexicontent');
// Get query size limit
$query = "SHOW VARIABLES LIKE 'max_allowed_packet'";
$db->setQuery($query);
$_dbvariable = $db->loadObject();
$max_allowed_packet = flexicontent_upload::parseByteLimit(@$_dbvariable->Value);
$max_allowed_packet = $max_allowed_packet ? $max_allowed_packet : 256 * 1024;
$query_lim = (int) (3 * $max_allowed_packet / 4);
//echo 'fail|'.$query_lim; exit;
// Get script max
$max_execution_time = ini_get("max_execution_time");
//echo 'fail|'.$max_execution_time; exit;
$query_count = 0;
$max_items_per_query = 100;
$max_items_per_query = $max_items_per_query > $items_per_call ? $items_per_call : $max_items_per_query;
$cnt = $itemcnt;
while ($cnt < count($itemids) && $cnt < $itemcnt + $items_per_call) {
$query_itemids = array_slice($itemids, $cnt, $max_items_per_query);
$cnt += $max_items_per_query;
// Get files
$data_query = "SELECT * " . " FROM #__flexicontent_files" . " WHERE id IN (" . implode(', ', $query_itemids) . ")";
$db->setQuery($data_query);
$file_data = $db->loadObjectList('id');
$vindex = array();
// For current item: Loop though all searchable fields according to their type
foreach ($file_data as $file_id => $file) {
$path = $file->secure ? COM_FLEXICONTENT_FILEPATH : COM_FLEXICONTENT_MEDIAPATH;
// JPATH_ROOT . DS . <media_path | file_path>
$file_path = $path . DS . $file->filename;
$file->size = !$file->url && file_exists($file_path) ? filesize($file_path) : 0;
$vindex[] = ' WHEN ' . $file->id . ' THEN ' . $file->size;
}
// Create query that will update/insert data into the DB
unset($query);
$query = 'UPDATE #__flexicontent_files ' . ' SET size = CASE id ' . implode('', $vindex) . ' END ' . ' WHERE id IN (' . implode(', ', $query_itemids) . ')';
$db->setQuery($query);
$db->execute();
$query_count++;
$elapsed_microseconds = round(1000000 * 10 * (microtime(true) - $start_microtime)) / 10;
$elapsed_seconds = $elapsed_microseconds / 1000000.0;
if ($elapsed_seconds > $max_execution_time / 3 || $elapsed_seconds > 5) {
break;
}
}
$elapsed_microseconds = round(1000000 * 10 * (microtime(true) - $start_microtime)) / 10;
if ($session->has($indexer . '_total_runtime', 'flexicontent')) {
$_total_runtime = $session->get($indexer . '_total_runtime', 0, 'flexicontent');
} else {
$_total_runtime = 0;
}
$_total_runtime += $elapsed_microseconds;
$session->set($indexer . '_total_runtime', $_total_runtime, 'flexicontent');
if ($session->has($indexer . '_total_queries', 'flexicontent')) {
$_total_queries = $session->get($indexer . '_total_queries', 0, 'flexicontent');
} else {
$_total_queries = 0;
}
$_total_queries += $query_count;
$session->set($indexer . '_total_queries', $_total_queries, 'flexicontent');
echo sprintf($cnt . ' | Server execution time: %.2f secs ', $_total_runtime / 1000000) . ' | Total DB updates: ' . $_total_queries;
exit;
}
