/** * Gets upload Limits * * @return array with limits * @since 3.0 */ static function getPHPuploadLimit() { $post_max = flexicontent_upload::parseByteLimit(ini_get('post_max_size')); $upload_max = flexicontent_upload::parseByteLimit(ini_get('upload_max_filesize')); if ($upload_max < $post_max) { $limit = array('value' => $upload_max, 'name' => 'upload_max_filesize'); } else { $limit = array('value' => $post_max, 'name' => 'post_max_size'); } // Sucosin limitation if (extension_loaded('suhosin')) { $post_max = flexicontent_upload::parseByteLimit(ini_get('suhosin.post.max_value_length')); if ($post_max < $limit['value']) { $limit = array('value' => $post_max, 'name' => 'suhosin.post.max_value_length'); } } return $limit; }
function index() { $start_microtime = microtime(true); $session = JFactory::getSession(); $db = JFactory::getDBO(); // Test indexing with limited memory //ini_set("memory_limit", "20M"); @ob_end_clean(); $search_prefix = JComponentHelper::getParams('com_flexicontent')->get('add_search_prefix') ? 'vvv' : ''; // SEARCH WORD Prefix $indexer = JRequest::getVar('indexer', 'advanced'); $rebuildmode = JRequest::getVar('rebuildmode', ''); $items_per_call = JRequest::getVar('items_per_call', 20); // Number of item to index per HTTP request $itemcnt = JRequest::getVar('itemcnt', 0); // Counter of items indexed so far, this is given via HTTP request // TAKE CARE: this code depends on countrows() to set session variables // Retrieve fields, that are assigned as (advanced/basic) searchable/filterable if ($rebuildmode == 'quick' && $indexer == 'advanced') { $nse_fields = $session->get($indexer . '_nse_fields', array(), 'flexicontent'); $nsp_fields = $session->get($indexer . '_nsp_fields', array(), 'flexicontent'); $fields = $session->get($indexer . '_fields', array(), 'flexicontent'); //echo 'fail|'; print_r(array_keys($fields)); exit; // Get the field ids of the fields removed from searching $del_fieldids = array_unique(array_merge(array_keys($nse_fields), array_keys($nsp_fields), array_keys($fields))); } else { $fields = $session->get($indexer . '_fields', array(), 'flexicontent'); //echo 'fail|'; print_r(array_keys($fields)); exit; } // Get the field ids of the searchable fields $fieldids = array_keys($fields); // Get fields that will have atomic search tables, (current for advanced index only) if ($indexer == 'advanced') { $filterables = FlexicontentFields::getSearchFields('id', $indexer, null, null, $_load_params = false, 0, $search_type = 'filter'); $filterables = array_keys($filterables); $filterables = array_flip($filterables); } else { $filterables = array(); } // Get items ids that have value for any of the searchable fields, but use session to avoid recalculation $itemids = $session->get($indexer . '_items_to_index', array(), 'flexicontent'); $_fields = array(); foreach ($fields as $field_id => $field) { // Clone field to avoid problems $_fields[$field_id] = clone $field; // Create field parameters if not already created if (empty($_fields[$field_id]->parameters)) { $_fields[$field_id]->parameters = new JRegistry($_fields[$field_id]->attribs); } } $fields = $_fields; // Get query size limit $query = "SHOW VARIABLES LIKE 'max_allowed_packet'"; $db->setQuery($query); $_dbvariable = $db->loadObject(); $max_allowed_packet = flexicontent_upload::parseByteLimit(@$_dbvariable->Value); $max_allowed_packet = $max_allowed_packet ? $max_allowed_packet : 256 * 1024; $query_lim = (int) (3 * $max_allowed_packet / 4); //echo 'fail|'.$query_lim; exit; // Get script max $max_execution_time = ini_get("max_execution_time"); //echo 'fail|'.$max_execution_time; exit; $query_count = 0; $max_items_per_query = 100; $max_items_per_query = $max_items_per_query > $items_per_call ? $items_per_call : $max_items_per_query; $cnt = $itemcnt; while ($cnt < count($itemids) && $cnt < $itemcnt + $items_per_call) { $query_itemids = array_slice($itemids, $cnt, $max_items_per_query); $cnt += $max_items_per_query; // Item is not needed, later and only if field uses item replacements then it will be loaded $item = null; // Items language is needed to do (if needed) special per language handling $lang_query = "SELECT id, language" . " FROM #__content AS i " . " WHERE id IN (" . implode(', ', $query_itemids) . ")"; $db->setQuery($lang_query); $items_data = $db->loadObjectList('id'); if ($indexer == 'basic') { $searchindex = array(); // Add all query itemids to searchindex array so that it will be cleared even if zero fields are indexed foreach ($query_itemids as $query_itemid) { $searchindex[$query_itemid] = array(); } } else { // This will hold the SQL inserting new advanced search records for multiple item/values $ai_query_vals = array(); $ai_query_vals_f = array(); // Current for advanced index only } // For current item: Loop though all searchable fields according to their type foreach ($fieldids as $fieldid) { // Must SHALLOW clone because we will be setting some properties , e.g. 'ai_query_vals', that we do not $field = clone $fields[$fieldid]; // Indicate multiple items per query $field->item_id = 0; $field->query_itemids = $query_itemids; $field->items_data = $items_data; // Includes item langyage, which may be used for special per language handling // Indicate that the indexing fuction should retrieve the values $values = null; // Add values to advanced search index $fieldname = $field->iscore ? 'core' : $field->field_type; if ($indexer == 'advanced') { FLEXIUtilities::call_FC_Field_Func($fieldname, 'onIndexAdvSearch', array(&$field, &$values, &$item)); //print_r($field->ai_query_vals); if (isset($field->ai_query_vals)) { foreach ($field->ai_query_vals as $query_val) { $ai_query_vals[] = $query_val; } if (isset($filterables[$field->id])) { // Current for advanced index only foreach ($field->ai_query_vals as $query_val) { $ai_query_vals_f[$field->id][] = $query_val; } } } //else echo "Not set for : ". $field->name; } else { if ($indexer == 'basic') { FLEXIUtilities::call_FC_Field_Func($fieldname, 'onIndexSearch', array(&$field, &$values, &$item)); foreach ($query_itemids as $query_itemid) { if (@$field->search[$query_itemid]) { $searchindex[$query_itemid][] = $field->search[$query_itemid]; } } } } } // Create query that will update/insert data into the DB unset($queries); // make sure it is not set above $queries = array(); if ($indexer == 'basic') { if (count($searchindex)) { // check for zero search index records $query_vals = ''; $query_ids = array(); // Start new query foreach ($searchindex as $query_itemid => $search_text) { if (strlen($query_vals) > $query_lim) { $query = "UPDATE #__flexicontent_items_ext SET search_index = CASE item_id " . $query_vals . " END " . " WHERE item_id IN (" . implode(',', $query_ids) . ")"; $queries[] = $query; $query_vals = ''; $query_ids = array(); // Start new query } $query_ids[] = $query_itemid; $_search_text = implode(' | ', $search_text); if ($search_prefix && $_search_text) { $_search_text = preg_replace('/(\\b[^\\s,\\.]+\\b)/u', $search_prefix . '$0', trim($_search_text)); } $query_vals .= " WHEN {$query_itemid} THEN " . $db->Quote($_search_text); } if (count($query_ids)) { $query = "UPDATE #__flexicontent_items_ext SET search_index = CASE item_id " . $query_vals . " END " . " WHERE item_id IN (" . implode(',', $query_ids) . ")"; $queries[] = $query; } } } else { if (count($ai_query_vals)) { // check for zero search index records $query_vals = ''; // Start new query foreach ($ai_query_vals as &$query_value) { $query_vals .= ($query_vals ? ',' : '') . $query_value; if (strlen($query_vals) > $query_lim) { $queries[] = "INSERT INTO #__flexicontent_advsearch_index " . " (field_id,item_id,extraid,search_index,value_id) VALUES " . $query_vals; $query_vals = ''; // Start new query } } unset($query_value); if (strlen($query_vals)) { $queries[] = "INSERT INTO #__flexicontent_advsearch_index " . " (field_id,item_id,extraid,search_index,value_id) VALUES " . $query_vals; } } foreach ($ai_query_vals_f as $_field_id => $_query_vals) { $query_vals = ''; // Start new query foreach ($_query_vals as &$query_value) { $query_vals .= ($query_vals ? ',' : '') . $query_value; if (strlen($query_vals) > $query_lim) { $queries[] = "INSERT INTO #__flexicontent_advsearch_index_field_" . $_field_id . " (field_id,item_id,extraid,search_index,value_id) VALUES " . $query_vals; $query_vals = ''; // Start new query } } if (strlen($query_vals)) { $queries[] = "INSERT INTO #__flexicontent_advsearch_index_field_" . $_field_id . " (field_id,item_id,extraid,search_index,value_id) VALUES " . $query_vals; $query_vals = ''; // Start new query } } } foreach ($queries as $query) { $db->setQuery($query); try { $db->execute(); } catch (RuntimeException $e) { echo "fail|" . $e->getMessage(); exit; } } $query_count += count($queries); $elapsed_microseconds = round(1000000 * 10 * (microtime(true) - $start_microtime)) / 10; $elapsed_seconds = $elapsed_microseconds / 1000000.0; if ($elapsed_seconds > $max_execution_time / 3 || $elapsed_seconds > 5) { break; } } // Check if items have finished, otherwise continue with -next- group of item ids if ($cnt >= count($itemids)) { // Reset dirty SEARCH properties of published fields to be: normal ON/OFF $set_clause = ' SET' . ($indexer == 'basic' ? ' issearch = CASE issearch WHEN 2 THEN 1 WHEN -1 THEN 0 ELSE issearch END' : ' isadvsearch = CASE isadvsearch WHEN 2 THEN 1 WHEN -1 THEN 0 ELSE isadvsearch END,' . ' isadvfilter = CASE isadvfilter WHEN 2 THEN 1 WHEN -1 THEN 0 ELSE isadvfilter END'); $query = 'UPDATE #__flexicontent_fields' . $set_clause . " WHERE published=1"; $db->setQuery($query); $db->execute(); // Force SEARCH properties of unpublished fields to be: normal OFF if ($indexer == 'basic') { $query = 'UPDATE #__flexicontent_fields SET issearch = 0 WHERE published=0'; $db->setQuery($query); $db->execute(); } else { $query = 'UPDATE #__flexicontent_fields SET isadvsearch = 0, isadvfilter = 0 WHERE published=0'; $db->setQuery($query); $db->execute(); } } if (!count($fieldids)) { echo 'fail|Index was only cleaned-up, <br/>since no <b>fields</b> were marked as: ' . '<br> -- ' . ($indexer == 'basic' ? 'Text Searchable (CONTENT LISTS)' : 'Text Searchable OR filterable (SEARCH VIEW)'); exit; } if (!count($itemids)) { echo 'fail|Index was only cleaned-up, <br/>since no <b>items</b> were found to have value for fields marked as: ' . '<br> -- ' . ($indexer == 'basic' ? 'Text Searchable (CONTENT LISTS)' : 'Text Searchable OR filterable (SEARCH VIEW)'); exit; } $elapsed_microseconds = round(1000000 * 10 * (microtime(true) - $start_microtime)) / 10; if ($session->has($indexer . '_total_runtime', 'flexicontent')) { $_total_runtime = $session->get($indexer . '_total_runtime', 0, 'flexicontent'); } else { $_total_runtime = 0; } $_total_runtime += $elapsed_microseconds; $session->set($indexer . '_total_runtime', $_total_runtime, 'flexicontent'); if ($session->has($indexer . '_total_queries', 'flexicontent')) { $_total_queries = $session->get($indexer . '_total_queries', 0, 'flexicontent'); } else { $_total_queries = 0; } $_total_queries += $query_count; $session->set($indexer . '_total_queries', $_total_queries, 'flexicontent'); echo sprintf($cnt . ' | Server execution time: %.2f secs ', $_total_runtime / 1000000) . ' | Total DB updates: ' . $_total_queries; exit; }
/** * Method to add flexi extended datas to standard content * * @params object the unassociated items rows * @params boolean add the records from the items_ext table * @return boolean * @since 1.5 */ function bindExtData($rows) { if (!$rows || !count($rows)) { return; } $app = JFactory::getApplication(); $jinput = $app->input; $search_prefix = $this->cparams->get('add_search_prefix') ? 'vvv' : ''; // SEARCH WORD Prefix $typeid = $jinput->get('typeid', 1, 'int'); $default_cat = $jinput->get('default_cat', 0, 'int'); $default_lang = flexicontent_html::getSiteDefaultLang(); // Get invalid cats, to avoid using them during binding, this is only done once $session = JFactory::getSession(); $badcats_fixed = $session->get('badcats', null, 'flexicontent'); if ($badcats_fixed === null) { // Correct non-existent main category in content table $query = 'UPDATE #__content as c ' . ' LEFT JOIN #__categories as cat ON c.catid=cat.id' . ' SET c.catid=' . $default_cat . ' WHERE cat.id IS NULL'; $this->_db->setQuery($query); $this->_db->execute(); $session->set('badcats_fixed', 1, 'flexicontent'); } // Calculate item data to be used for current bind STEP $catrel = array(); foreach ($rows as $row) { $row_catid = (int) $row->catid; $catrel[] = '(' . $row_catid . ', ' . (int) $row->id . ')'; // append the text property to the object if (JString::strlen($row->fulltext) > 1) { $row->text_stripped = $row->introtext . '<hr id="system-readmore" />' . $row->fulltext; } else { $row->text_stripped = flexicontent_html::striptagsandcut($row->introtext); } } // Insert main category-item relation via single query $catrel = implode(', ', $catrel); $query = "INSERT INTO #__flexicontent_cats_item_relations (`catid`, `itemid`) " . " VALUES " . $catrel . " ON DUPLICATE KEY UPDATE ordering=ordering"; $this->_db->setQuery($query); $this->_db->execute(); $query = "SHOW VARIABLES LIKE 'max_allowed_packet'"; $this->_db->setQuery($query); $_dbvariable = $this->_db->loadObject(); $max_allowed_packet = flexicontent_upload::parseByteLimit(@$_dbvariable->Value); $max_allowed_packet = $max_allowed_packet ? $max_allowed_packet : 256 * 1024; $query_lim = (int) (3 * $max_allowed_packet / 4); // Insert items_ext datas, // NOTE: we will not use a single query for creating multiple records, instead we will create only e.g. 100 at once, // because of the column search_index which can be quite long $itemext = array(); $id_arr = array(); $row_count = count($rows); $n = 0; $i = 0; $query_len = 0; foreach ($rows as $row) { $ilang = $row->language ? $row->language : $default_lang; if ($search_prefix) { $_search_index = preg_replace('/(\\b[^\\s,\\.]+\\b)/u', $search_prefix . '$0', $row->title . ' | ' . $row->text_stripped); } else { $_search_index = $row->title . ' | ' . $row->text_stripped; } $itemext[$i] = '(' . (int) $row->id . ', ' . $typeid . ', ' . $this->_db->Quote($ilang) . ', ' . $this->_db->Quote($_search_index) . ', 0)'; $id_arr[$i] = (int) $row->id; $query_len += strlen($itemext[$i]) + 2; // Sum of query length so far $n++; $i++; if ($n % 101 == 0 || $n == $row_count || $query_len > $query_lim) { $itemext_list = implode(', ', $itemext); $query = "INSERT INTO #__flexicontent_items_ext (`item_id`, `type_id`, `language`, `search_index`, `lang_parent_id`)" . " VALUES " . $itemext_list . " ON DUPLICATE KEY UPDATE type_id=VALUES(type_id), language=VALUES(language), search_index=VALUES(search_index)"; $this->_db->setQuery($query); $this->_db->execute(); // reset the item array $itemext = array(); $query = "UPDATE #__flexicontent_items_tmp" . " SET type_id=" . $typeid . " WHERE id IN(" . implode(',', $id_arr) . ")"; $this->_db->setQuery($query); $this->_db->execute(); // reset the item id array $id_arr = array(); $i = 0; // reset sub-counter, and query length $query_len = 0; } } // Update temporary item data $this->updateItemCountingData($rows); }
function index() { $start_microtime = microtime(true); $session = JFactory::getSession(); $db = JFactory::getDBO(); @ob_end_clean(); $indexer = JRequest::getVar('indexer', 'fileman_default'); $rebuildmode = JRequest::getVar('rebuildmode', ''); $items_per_call = JRequest::getVar('items_per_call', 20); // Number of item to index per HTTP request $itemcnt = JRequest::getVar('itemcnt', 0); // Counter of items indexed so far, this is given via HTTP request // Actions according to rebuildmode if ($indexer != 'fileman_default') { die("'rebuildmode': '" . $rebuildmode . "'. not supported"); } // Get items ids that have value for any of the searchable fields, but use session to avoid recalculation $itemids = $session->get($indexer . '_items_to_index', array(), 'flexicontent'); // Get query size limit $query = "SHOW VARIABLES LIKE 'max_allowed_packet'"; $db->setQuery($query); $_dbvariable = $db->loadObject(); $max_allowed_packet = flexicontent_upload::parseByteLimit(@$_dbvariable->Value); $max_allowed_packet = $max_allowed_packet ? $max_allowed_packet : 256 * 1024; $query_lim = (int) (3 * $max_allowed_packet / 4); //echo 'fail|'.$query_lim; exit; // Get script max $max_execution_time = ini_get("max_execution_time"); //echo 'fail|'.$max_execution_time; exit; $query_count = 0; $max_items_per_query = 100; $max_items_per_query = $max_items_per_query > $items_per_call ? $items_per_call : $max_items_per_query; $cnt = $itemcnt; while ($cnt < count($itemids) && $cnt < $itemcnt + $items_per_call) { $query_itemids = array_slice($itemids, $cnt, $max_items_per_query); $cnt += $max_items_per_query; // Get files $data_query = "SELECT * " . " FROM #__flexicontent_files" . " WHERE id IN (" . implode(', ', $query_itemids) . ")"; $db->setQuery($data_query); $file_data = $db->loadObjectList('id'); $vindex = array(); // For current item: Loop though all searchable fields according to their type foreach ($file_data as $file_id => $file) { $path = $file->secure ? COM_FLEXICONTENT_FILEPATH : COM_FLEXICONTENT_MEDIAPATH; // JPATH_ROOT . DS . <media_path | file_path> $file_path = $path . DS . $file->filename; $file->size = !$file->url && file_exists($file_path) ? filesize($file_path) : 0; $vindex[] = ' WHEN ' . $file->id . ' THEN ' . $file->size; } // Create query that will update/insert data into the DB unset($query); $query = 'UPDATE #__flexicontent_files ' . ' SET size = CASE id ' . implode('', $vindex) . ' END ' . ' WHERE id IN (' . implode(', ', $query_itemids) . ')'; $db->setQuery($query); $db->execute(); $query_count++; $elapsed_microseconds = round(1000000 * 10 * (microtime(true) - $start_microtime)) / 10; $elapsed_seconds = $elapsed_microseconds / 1000000.0; if ($elapsed_seconds > $max_execution_time / 3 || $elapsed_seconds > 5) { break; } } $elapsed_microseconds = round(1000000 * 10 * (microtime(true) - $start_microtime)) / 10; if ($session->has($indexer . '_total_runtime', 'flexicontent')) { $_total_runtime = $session->get($indexer . '_total_runtime', 0, 'flexicontent'); } else { $_total_runtime = 0; } $_total_runtime += $elapsed_microseconds; $session->set($indexer . '_total_runtime', $_total_runtime, 'flexicontent'); if ($session->has($indexer . '_total_queries', 'flexicontent')) { $_total_queries = $session->get($indexer . '_total_queries', 0, 'flexicontent'); } else { $_total_queries = 0; } $_total_queries += $query_count; $session->set($indexer . '_total_queries', $_total_queries, 'flexicontent'); echo sprintf($cnt . ' | Server execution time: %.2f secs ', $_total_runtime / 1000000) . ' | Total DB updates: ' . $_total_queries; exit; }