/**
* Determine if a module is FTS enabled.
*
* @param $module
* @return bool
*/
protected function isModuleFtsEnabled($module)
{
return SugarSearchEngineMetadataHelper::isModuleFtsEnabled($module);
}
/**
* The collector method for modules. Gets metadata for all of the module specific data
*
* @param $moduleName The name of the module to collect metadata about.
* @param MetaDataContextInterface|null $context Metadata context
* @return array An array of hashes containing the metadata. Empty arrays are
* returned in the case of no metadata.
*/
public function getModuleData($moduleName, MetaDataContextInterface $context = null)
{
require_once 'include/SugarSearchEngine/SugarSearchEngineMetadataHelper.php';
$vardefs = $this->getVarDef($moduleName);
if (!empty($vardefs['fields']) && is_array($vardefs['fields'])) {
require_once 'include/MassUpdate.php';
$vardefs['fields'] = MassUpdate::setMassUpdateFielddefs($vardefs['fields'], $moduleName);
}
$data['fields'] = isset($vardefs['fields']) ? $vardefs['fields'] : array();
// Add the _hash for the fields array
$data['fields']['_hash'] = md5(serialize($data['fields']));
$data['nameFormat'] = isset($vardefs['name_format_map']) ? $vardefs['name_format_map'] : null;
$data['views'] = $this->getModuleViews($moduleName, $context);
$data['datas'] = $this->getModuleDatas($moduleName);
$data['layouts'] = $this->getModuleLayouts($moduleName);
$data['fieldTemplates'] = $this->getModuleFields($moduleName);
$data['menu'] = $this->getModuleMenu($moduleName);
$data['config'] = $this->getModuleConfig($moduleName);
$data['filters'] = $this->getModuleFilters($moduleName);
// Indicate whether Module Has duplicate checking enabled --- Rules must exist and Enabled flag must be set
$data['dupCheckEnabled'] = isset($vardefs['duplicate_check']) && isset($vardefs['duplicate_check']['enabled']) && $vardefs['duplicate_check']['enabled'] === true;
// Indicate whether a Module has activity stream enabled
$data['activityStreamEnabled'] = ActivityQueueManager::isEnabledForModule($moduleName);
$data['ftsEnabled'] = SugarSearchEngineMetadataHelper::isModuleFtsEnabled($moduleName);
// TODO we need to have this kind of information on the module itself not hacked around on globals
$data['isBwcEnabled'] = in_array($moduleName, $GLOBALS['bwcModules']);
$seed = BeanFactory::newBean($moduleName);
$data['globalSearchEnabled'] = $this->getGlobalSearchEnabled($seed, $vardefs, $this->platforms[0]);
if (!empty($seed)) {
$favoritesEnabled = $seed->isFavoritesEnabled() !== false ? true : false;
$data['favoritesEnabled'] = $favoritesEnabled;
}
// Currently no way to disable following
// But this flag is here in case we add that feature in the future
$data['followingEnabled'] = true;
$data["_hash"] = $this->hashChunk($data);
return $data;
}
/**
* This function is used to hand off the global search to the FTS Search Emgine
* @param $api ServiceBase The API class of the request
* @param $args array The arguments array passed in from the API
* @param $searchEngine SugarSearchEngine The SugarSpot search engine created using the Factory in the caller
* @param $options array An array of options to pass through to the search engine, they get translated to the $searchOptions array so you can see exactly what gets passed through
* @return array Two elements, 'records' the list of returned records formatted through FormatBean, and 'next_offset' which will indicate to the user if there are additional records to be returned.
*/
protected function globalSearchFullText(ServiceBase $api, array $args, SugarSearchEngineAbstractBase $searchEngine, array $options)
{
$options['append_wildcard'] = 1;
if (empty($options['moduleList'])) {
require_once 'modules/ACL/ACLController.php';
$moduleList = SugarSearchEngineMetadataHelper::getSystemEnabledFTSModules();
// filter based on User Access if Blank
$ACL = new ACLController();
// moduleList is passed by reference
$ACL->filterModuleList($moduleList);
$options['moduleList'] = $moduleList;
}
if (!empty($options['searchFields'])) {
$customWhere = array();
foreach ($options['moduleList'] as $module) {
$seed = BeanFactory::getBean($module);
$fields = array_keys($seed->field_defs);
$existingfields = array_intersect($fields, $options['searchFields']);
if (!empty($existingfields)) {
foreach ($existingfields as $field) {
if (empty($seed->field_defs[$field]['full_text_search'])) {
continue;
}
$prefix = $seed->module_name;
if (!isset($seed->field_defs[$field]['source']) || $seed->field_defs[$field]['source'] != 'non-db') {
$customWhere[] = "{$prefix}.{$field}";
}
}
}
}
$options['searchFields'] = $customWhere;
}
$options['moduleFilter'] = $options['moduleList'];
$results = $searchEngine->search($options['query'], $options['offset'], $options['limit'], $options);
$returnedRecords = array();
$total = 0;
$api->action = 'list';
if (is_object($results)) {
foreach ($results as $result) {
$record = BeanFactory::retrieveBean($result->getModule(), $result->getId());
// if we can't get the bean skip it
if (!$record) {
continue;
}
$module = $record->module_dir;
// Need to override the filter arg so that it looks like something formatBean expects
if (!empty($options['fieldFilters'][$module])) {
$moduleFields = $options['fieldFilters'][$module];
} else {
if (!empty($options['fieldFilters']['_default'])) {
$moduleFields = $options['fieldFilters']['_default'];
} else {
$moduleFields = array();
}
}
if (!empty($moduleFields) && !in_array('id', $moduleFields)) {
$moduleFields[] = 'id';
}
$moduleArgs['fields'] = implode(',', $moduleFields);
$formattedRecord = $this->formatBean($api, $moduleArgs, $record);
$formattedRecord['_module'] = $module;
// The SQL based search engine doesn't know how to score records, so set it to 1
$formattedRecord['_search']['score'] = $result->getScore();
//Add highlighted text
$formattedRecord['_search']['highlighted'] = $result->getHighlightedHitText();
$returnedRecords[] = $formattedRecord;
}
$total = $results->getTotalHits();
}
if ($total > $options['limit'] + $options['offset']) {
$nextOffset = $options['offset'] + $options['limit'];
} else {
$nextOffset = -1;
}
return array('next_offset' => $nextOffset, 'records' => $returnedRecords);
}
/**
*
* This function creates a full mapping for all modules.
* index must exist before calling this function.
*
*/
public function setFullMapping()
{
$allModules = SugarSearchEngineMetadataHelper::retrieveFtsEnabledFieldsForAllModules();
// if the index already exists, is there a way to create mapping for multiple modules at once?
// for now, create one mapping for a module at a time
foreach ($allModules as $name => $module) {
$this->setFieldMapping($name, $module);
}
}
/**
* Remove the cached unified_search_modules.php file
*/
public function clearSearchCache()
{
global $mod_strings, $sugar_config;
if ($this->show_output) {
echo "<h3>{$mod_strings['LBL_QR_CLEARSEARCH']}</h3>";
}
// clear sugar_cache backend for SugarSearchEngine
SugarSearchEngineMetadataHelper::clearCache();
// Clear the cache file AFTER the cache clear, as it will be rebuilt by
// clearCache otherwise
UnifiedSearchAdvanced::unlinkUnifiedSearchModulesFile();
}
/**
*
* Load facet definitions from enabled FTS module fieldDefs
* @param string $module
* @return array
*/
protected function loadFacetDefs($module)
{
if (isset($this->facetDefs[$module])) {
return $this->facetDefs[$module];
}
$this->facetDefs[$module] = array();
$fieldDefs = SugarSearchEngineMetadataHelper::retrieveFtsEnabledFieldsPerModule($module);
foreach ($fieldDefs as $fieldName => $fieldDef) {
// skip non-facet defs
if (empty($fieldDef['full_text_search']['facet'])) {
continue;
}
$facet = $fieldDef['full_text_search']['facet'];
// the type needs to be defined to be valid
if (is_array($facet) && !empty($facet['type'])) {
// set empty options array if nothing specified
if (empty($facet['options']) || !is_array($facet['options'])) {
$facet['options'] = array();
}
// set defaults for ui_type and label
$facet['ui_type'] = empty($facet['ui_type']) ? 'checkbox' : $facet['ui_type'];
$facet['label'] = empty($facet['label']) ? 'LBL_FACET_NOT_SET' : $facet['label'];
$this->facetDefs[$module][$fieldName] = $facet;
}
}
return $this->facetDefs[$module];
}
/**
* Get the enabled and disabled modules for the datatable
*
* @param $moduleFilter array Requested modules for search
* @param $disabledModules array Requested modules for disable in search
* @return array
*/
protected function getFilterModules($moduleFilter, $disabledModules)
{
$filteredEnabled = SugarSearchEngineMetadataHelper::getUserEnabledFTSModules();
$userDisabled = $GLOBALS['current_user']->getPreference('fts_disabled_modules');
$userDisabled = explode(",", $userDisabled);
// Filter by System enabled FTS modules
$systemEnabledModules = SugarSearchEngineMetadataHelper::getSystemEnabledFTSModules();
$userDisabled = array_intersect_key($systemEnabledModules, array_flip($userDisabled));
$filteredEnabled = array_intersect_key($systemEnabledModules, array_flip($filteredEnabled));
$userDisabled = $this->translateModulesList($userDisabled);
$filteredEnabled = $this->translateModulesList($filteredEnabled);
sort($filteredEnabled);
if (!empty($moduleFilter)) {
foreach ($filteredEnabled as $key => $info) {
if (!in_array($info['module'], $moduleFilter) && in_array($info['module'], $disabledModules)) {
unset($filteredEnabled[$key]);
// its not enabled, its disabled
$userDisabled = $info;
}
}
}
return array('enabled' => $filteredEnabled, 'disabled' => $userDisabled);
}
/**
* This function returns an array of fields that can be passed to search engine.
* @param Array $options
* @param boolean $prefixed Add module name prefix (i.e. Contacts.first_name)
* @return Array array of fields
*/
protected function getSearchFields($options, $prefixed = true)
{
$fields = array();
// determine list of modules/fields
$allFieldDefs = array();
if (!empty($options['moduleFilter'])) {
foreach ($options['moduleFilter'] as $module) {
$allFieldDefs[$module] = SugarSearchEngineMetadataHelper::retrieveFtsEnabledFieldsPerModule($module);
}
} else {
$allFieldDefs = SugarSearchEngineMetadataHelper::retrieveFtsEnabledFieldsForAllModules();
}
// build list of fields with optional boost values (i.e. Accouns.name^3)
foreach ($allFieldDefs as $module => $fieldDefs) {
foreach ($fieldDefs as $fieldName => $fieldDef) {
// skip non-supported field types
$ftsType = $this->mapper->getFtsTypeFromDef($fieldDef);
if (!$ftsType || in_array($ftsType['type'], $this->ignoreSearchTypes)) {
$this->logger->debug("Elastic: Ignoring unsupported type in query for {$module}/{$fieldName}");
continue;
}
// base field name
if ($prefixed) {
$fieldName = $module . '.' . $fieldName;
}
// To enable a field for user search we require a boost value. There may be other fields
// we index into Elastic but which should not be user searchable. We use the boost value
// being set or not to distinguish between both scenarios. For example for extended facets
// and related fields we can store additional fields or non analyzed data. While we need
// those fields being indexed, we do not want the user to be able to hit those when
// performing a search.
if (empty($fieldDef['full_text_search']['boost'])) {
$this->logger->debug("Elastic: skipping {$module}/{$fieldName} for search field (no boost set)");
continue;
} else {
if (!empty($options['addSearchBoosts'])) {
$fieldName .= '^' . $fieldDef['full_text_search']['boost'];
}
$fields[] = $fieldName;
}
}
}
return $fields;
}
/**
* Main function that handles the indexing of a bean and is called by the job queue system.
*
* @param $data
*/
public function run($module)
{
$serverOK = $this->updateFTSServerStatus();
if ($serverOK != true) {
$GLOBALS['log']->fatal('FTS Server is down, postponing the job for full index.');
$this->schedulerJob->postponeJob(null, $this->postpone_job_time);
return true;
}
$this->db->commit();
$GLOBALS['log']->info("Going to index all records in module {$module} ");
$startTime = microtime(true);
$fieldDefinitions = SugarSearchEngineMetadataHelper::retrieveFtsEnabledFieldsPerModule($module);
if (!empty($fieldDefinitions)) {
$count = $this->indexRecords($module, $fieldDefinitions);
} else {
$GLOBALS['log']->fatal(sprintf('Fields of %s are not enabled for Full Text Search', $module));
$this->schedulerJob->resolveJob(SchedulersJob::JOB_STATUS_DONE);
return true;
}
if ($count == -1) {
$GLOBALS['log']->fatal('FTS failed to index records, postponing job for next cron');
$this->schedulerJob->postponeJob(null, $this->postpone_job_time);
return true;
}
// stats logging
$totalTime = number_format(round(microtime(true) - $startTime, 2), 2);
$avgRecs = $count != 0 && $totalTime != 0 ? number_format(round($count / $totalTime, 2), 2) : 0;
$GLOBALS['log']->info(sprintf("FTS Consumer %s processed %s record(s) in %s secs, records per sec: %s", $this->schedulerJob->name, $count, $totalTime, $avgRecs));
// Mark the job that as pending so we can be invoked later again - this is considered to be a persistent job
$this->schedulerJob->postponeJob(null, $this->postpone_job_time);
return true;
}