/**
* Returns an array of variants.
*
* With no arguments, returns all variants
*
* With a classname as the first argument, returns the variants that apply to that class
* (optionally including subclasses)
*
* @static
* @param string $class - The class name to get variants for
* @param bool $includeSubclasses - True if variants should be included if they apply to at least one subclass of $class
* @return array - An array of (string)$variantClassName => (Object)$variantInstance pairs
*/
public static function variants($class = null, $includeSubclasses = true)
{
if (!$class) {
if (self::$variants === null) {
$classes = ClassInfo::subclassesFor('SearchVariant');
$concrete = array();
foreach ($classes as $variantclass) {
$ref = new ReflectionClass($variantclass);
if ($ref->isInstantiable()) {
$variant = singleton($variantclass);
if ($variant->appliesToEnvironment()) {
$concrete[$variantclass] = $variant;
}
}
}
self::$variants = $concrete;
}
return self::$variants;
} else {
$key = $class . '!' . $includeSubclasses;
if (!isset(self::$class_variants[$key])) {
self::$class_variants[$key] = array();
foreach (self::variants() as $variantclass => $instance) {
if ($instance->appliesTo($class, $includeSubclasses)) {
self::$class_variants[$key][$variantclass] = $instance;
}
}
}
return self::$class_variants[$key];
}
}
/**
* Fulltextsearch module doesn't yet support facets very well, so I've just copied this function here so
* we have access to the results. I'd prefer to modify it minimally so we can eventually get rid of it
* once they add faceting or hooks to get directly at the returned response.
*
* @param SearchQuery $query
* @param integer $offset
* @param integer $limit
* @param Array $params Extra request parameters passed through to Solr
* @param array $facetSpec - Added for ShopSearch so we can process the facets
* @return ArrayData Map with the following keys:
* - 'Matches': ArrayList of the matched object instances
*/
public function search(SearchQuery $query, $offset = -1, $limit = -1, $params = array(), $facetSpec = array())
{
$service = $this->getService();
SearchVariant::with(count($query->classes) == 1 ? $query->classes[0]['class'] : null)->call('alterQuery', $query, $this);
$q = array();
$fq = array();
// Build the search itself
foreach ($query->search as $search) {
$text = $search['text'];
preg_match_all('/"[^"]*"|\\S+/', $text, $parts);
$fuzzy = $search['fuzzy'] ? '~' : '';
foreach ($parts[0] as $part) {
$fields = isset($search['fields']) ? $search['fields'] : array();
if (isset($search['boost'])) {
$fields = array_merge($fields, array_keys($search['boost']));
}
if ($fields) {
$searchq = array();
foreach ($fields as $field) {
$boost = isset($search['boost'][$field]) ? '^' . $search['boost'][$field] : '';
$searchq[] = "{$field}:" . $part . $fuzzy . $boost;
}
$q[] = '+(' . implode(' OR ', $searchq) . ')';
} else {
$q[] = '+' . $part . $fuzzy;
}
}
}
// Filter by class if requested
$classq = array();
foreach ($query->classes as $class) {
if (!empty($class['includeSubclasses'])) {
$classq[] = 'ClassHierarchy:' . $class['class'];
} else {
$classq[] = 'ClassName:' . $class['class'];
}
}
if ($classq) {
$fq[] = '+(' . implode(' ', $classq) . ')';
}
// Filter by filters
foreach ($query->require as $field => $values) {
$requireq = array();
foreach ($values as $value) {
if ($value === SearchQuery::$missing) {
$requireq[] = "(*:* -{$field}:[* TO *])";
} elseif ($value === SearchQuery::$present) {
$requireq[] = "{$field}:[* TO *]";
} elseif ($value instanceof SearchQuery_Range) {
$start = $value->start;
if ($start === null) {
$start = '*';
}
$end = $value->end;
if ($end === null) {
$end = '*';
}
$requireq[] = "{$field}:[{$start} TO {$end}]";
} else {
$requireq[] = $field . ':"' . $value . '"';
}
}
$fq[] = '+(' . implode(' ', $requireq) . ')';
}
foreach ($query->exclude as $field => $values) {
$excludeq = array();
$missing = false;
foreach ($values as $value) {
if ($value === SearchQuery::$missing) {
$missing = true;
} elseif ($value === SearchQuery::$present) {
$excludeq[] = "{$field}:[* TO *]";
} elseif ($value instanceof SearchQuery_Range) {
$start = $value->start;
if ($start === null) {
$start = '*';
}
$end = $value->end;
if ($end === null) {
$end = '*';
}
$excludeq[] = "{$field}:[{$start} TO {$end}]";
} else {
$excludeq[] = $field . ':"' . $value . '"';
}
}
$fq[] = ($missing ? "+{$field}:[* TO *] " : '') . '-(' . implode(' ', $excludeq) . ')';
}
// if(!headers_sent()) {
// if ($q) header('X-Query: '.implode(' ', $q));
// if ($fq) header('X-Filters: "'.implode('", "', $fq).'"');
// }
if ($offset == -1) {
$offset = $query->start;
}
if ($limit == -1) {
$limit = $query->limit;
}
if ($limit == -1) {
$limit = SearchQuery::$default_page_size;
}
$params = array_merge($params, array('fq' => implode(' ', $fq)));
$res = $service->search($q ? implode(' ', $q) : '*:*', $offset, $limit, $params, Apache_Solr_Service::METHOD_POST);
//Debug::dump($res);
$results = new ArrayList();
if ($res->getHttpStatus() >= 200 && $res->getHttpStatus() < 300) {
foreach ($res->response->docs as $doc) {
$result = DataObject::get_by_id($doc->ClassName, $doc->ID);
if ($result) {
$results->push($result);
// Add highlighting (optional)
$docId = $doc->_documentid;
if ($res->highlighting && $res->highlighting->{$docId}) {
// TODO Create decorator class for search results rather than adding arbitrary object properties
// TODO Allow specifying highlighted field, and lazy loading
// in case the search API needs another query (similar to SphinxSearchable->buildExcerpt()).
$combinedHighlights = array();
foreach ($res->highlighting->{$docId} as $field => $highlights) {
$combinedHighlights = array_merge($combinedHighlights, $highlights);
}
// Remove entity-encoded U+FFFD replacement character. It signifies non-displayable characters,
// and shows up as an encoding error in browsers.
$result->Excerpt = DBField::create_field('HTMLText', str_replace('�', '', implode(' ... ', $combinedHighlights)));
}
}
}
$numFound = $res->response->numFound;
} else {
$numFound = 0;
}
$ret = array();
$ret['Matches'] = new PaginatedList($results);
$ret['Matches']->setLimitItems(false);
// Tell PaginatedList how many results there are
$ret['Matches']->setTotalItems($numFound);
// Results for current page start at $offset
$ret['Matches']->setPageStart($offset);
// Results per page
$ret['Matches']->setPageLength($limit);
// Facets
//Debug::dump($res);
if (isset($res->facet_counts->facet_fields)) {
$ret['Facets'] = $this->buildFacetResults($res->facet_counts->facet_fields, $facetSpec);
}
// Suggestions (requires custom setup, assumes spellcheck.collate=true)
if (isset($res->spellcheck->suggestions->collation)) {
$ret['Suggestion'] = $res->spellcheck->suggestions->collation;
}
return new ArrayData($ret);
}
/**
* Given a class, object id, set of stateful ids and a list of changed fields (in a special format),
* return what statefulids need updating in this index
*
* Internal function used by SearchUpdater.
*
* @param $class
* @param $id
* @param $statefulids
* @param $fields
* @return array
*/
public function getDirtyIDs($class, $id, $statefulids, $fields)
{
$dirty = array();
// First, if this object is directly contained in the index, add it
foreach ($this->classes as $searchclass => $options) {
if ($searchclass == $class || $options['include_children'] && is_subclass_of($class, $searchclass)) {
$base = ClassInfo::baseDataClass($searchclass);
$dirty[$base] = array();
foreach ($statefulids as $statefulid) {
$key = serialize($statefulid);
$dirty[$base][$key] = $statefulid;
}
}
}
$current = SearchVariant::current_state();
// Then, for every derived field
foreach ($this->getDerivedFields() as $derivation) {
// If the this object is a subclass of any of the classes we want a field from
if (!SearchIntrospection::is_subclass_of($class, $derivation['classes'])) {
continue;
}
if (!array_intersect_key($fields, $derivation['fields'])) {
continue;
}
foreach (SearchVariant::reindex_states($class, false) as $state) {
SearchVariant::activate_state($state);
$ids = array($id);
foreach ($derivation['chain'] as $step) {
if ($step['through'] == 'has_one') {
$sql = new SQLQuery('"ID"', '"' . $step['class'] . '"', '"' . $step['foreignkey'] . '" IN (' . implode(',', $ids) . ')');
singleton($step['class'])->extend('augmentSQL', $sql);
$ids = $sql->execute()->column();
} else {
if ($step['through'] == 'has_many') {
$sql = new SQLQuery('"' . $step['class'] . '"."ID"', '"' . $step['class'] . '"', '"' . $step['otherclass'] . '"."ID" IN (' . implode(',', $ids) . ')');
$sql->addInnerJoin($step['otherclass'], '"' . $step['class'] . '"."ID" = "' . $step['otherclass'] . '"."' . $step['foreignkey'] . '"');
singleton($step['class'])->extend('augmentSQL', $sql);
$ids = $sql->execute()->column();
}
}
}
SearchVariant::activate_state($current);
if ($ids) {
$base = $derivation['base'];
if (!isset($dirty[$base])) {
$dirty[$base] = array();
}
foreach ($ids as $id) {
$statefulid = array('id' => $id, 'state' => $state);
$key = serialize($statefulid);
$dirty[$base][$key] = $statefulid;
}
}
}
}
return $dirty;
}
/**
* @param SearchQuery $query
* @param integer $offset
* @param integer $limit
* @param array $params Extra request parameters passed through to Solr
* @return ArrayData Map with the following keys:
* - 'Matches': ArrayList of the matched object instances
*/
public function search(SearchQuery $query, $offset = -1, $limit = -1, $params = array())
{
$service = $this->getService();
$searchClass = count($query->classes) == 1 ? $query->classes[0]['class'] : null;
SearchVariant::with($searchClass)->call('alterQuery', $query, $this);
$q = array();
// Query
$fq = array();
// Filter query
$qf = array();
// Query fields
$hlq = array();
// Highlight query
// Build the search itself
$q = $this->getQueryComponent($query, $hlq);
// If using boosting, set the clean term separately for highlighting.
// See https://issues.apache.org/jira/browse/SOLR-2632
if (array_key_exists('hl', $params) && !array_key_exists('hl.q', $params)) {
$params['hl.q'] = implode(' ', $hlq);
}
// Filter by class if requested
$classq = array();
foreach ($query->classes as $class) {
if (!empty($class['includeSubclasses'])) {
$classq[] = 'ClassHierarchy:' . $class['class'];
} else {
$classq[] = 'ClassName:' . $class['class'];
}
}
if ($classq) {
$fq[] = '+(' . implode(' ', $classq) . ')';
}
// Filter by filters
$fq = array_merge($fq, $this->getFiltersComponent($query));
// Prepare query fields unless specified explicitly
if (isset($params['qf'])) {
$qf = $params['qf'];
} else {
$qf = $this->getQueryFields();
}
if (is_array($qf)) {
$qf = implode(' ', $qf);
}
if ($qf) {
$params['qf'] = $qf;
}
if (!headers_sent() && !Director::isLive()) {
if ($q) {
header('X-Query: ' . implode(' ', $q));
}
if ($fq) {
header('X-Filters: "' . implode('", "', $fq) . '"');
}
if ($qf) {
header('X-QueryFields: ' . $qf);
}
}
if ($offset == -1) {
$offset = $query->start;
}
if ($limit == -1) {
$limit = $query->limit;
}
if ($limit == -1) {
$limit = SearchQuery::$default_page_size;
}
$params = array_merge($params, array('fq' => implode(' ', $fq)));
$res = $service->search($q ? implode(' ', $q) : '*:*', $offset, $limit, $params, Apache_Solr_Service::METHOD_POST);
$results = new ArrayList();
if ($res->getHttpStatus() >= 200 && $res->getHttpStatus() < 300) {
foreach ($res->response->docs as $doc) {
$result = DataObject::get_by_id($doc->ClassName, $doc->ID);
if ($result) {
$results->push($result);
// Add highlighting (optional)
$docId = $doc->_documentid;
if ($res->highlighting && $res->highlighting->{$docId}) {
// TODO Create decorator class for search results rather than adding arbitrary object properties
// TODO Allow specifying highlighted field, and lazy loading
// in case the search API needs another query (similar to SphinxSearchable->buildExcerpt()).
$combinedHighlights = array();
foreach ($res->highlighting->{$docId} as $field => $highlights) {
$combinedHighlights = array_merge($combinedHighlights, $highlights);
}
// Remove entity-encoded U+FFFD replacement character. It signifies non-displayable characters,
// and shows up as an encoding error in browsers.
$result->Excerpt = DBField::create_field('HTMLText', str_replace('�', '', implode(' ... ', $combinedHighlights)));
}
}
}
$numFound = $res->response->numFound;
} else {
$numFound = 0;
}
$ret = array();
$ret['Matches'] = new PaginatedList($results);
$ret['Matches']->setLimitItems(false);
// Tell PaginatedList how many results there are
$ret['Matches']->setTotalItems($numFound);
// Results for current page start at $offset
$ret['Matches']->setPageStart($offset);
// Results per page
$ret['Matches']->setPageLength($limit);
// Include spellcheck and suggestion data. Requires spellcheck=true in $params
if (isset($res->spellcheck)) {
// Expose all spellcheck data, for custom handling.
$ret['Spellcheck'] = $res->spellcheck;
// Suggestions. Requires spellcheck.collate=true in $params
if (isset($res->spellcheck->suggestions->collation)) {
// Extract string suggestion
$suggestion = $this->getCollatedSuggestion($res->spellcheck->suggestions->collation);
// The collation, including advanced query params (e.g. +), suitable for making another query programmatically.
$ret['Suggestion'] = $suggestion;
// A human friendly version of the suggestion, suitable for 'Did you mean $SuggestionNice?' display.
$ret['SuggestionNice'] = $this->getNiceSuggestion($suggestion);
// A string suitable for appending to an href as a query string.
// For example <a href="http://example.com/search?q=$SuggestionQueryString">$SuggestionNice</a>
$ret['SuggestionQueryString'] = $this->getSuggestionQueryString($suggestion);
}
}
return new ArrayData($ret);
}
/**
* Clear all records of the given class in the current state ONLY.
*
* Optionally delete from a given group (where the group is defined as the ID % total groups)
*
* @param SolrIndex $indexInstance Index instance
* @param string $class Class name
* @param int $groups Number of groups, if clearing from a striped group
* @param int $group Group number, if clearing from a striped group
*/
protected function clearRecords(SolrIndex $indexInstance, $class, $groups = null, $group = null)
{
// Clear by classname
$conditions = array("+(ClassHierarchy:{$class})");
// If grouping, delete from this group only
if ($groups) {
$conditions[] = "+_query_:\"{!frange l={$group} u={$group}}mod(ID, {$groups})\"";
}
// Also filter by state (suffix on document ID)
$query = new SearchQuery();
SearchVariant::with($class)->call('alterQuery', $query, $indexInstance);
if ($query->isfiltered()) {
$conditions = array_merge($conditions, $indexInstance->getFiltersComponent($query));
}
// Invoke delete on index
$deleteQuery = implode(' ', $conditions);
$indexInstance->getService()->deleteByQuery($deleteQuery);
}
/**
* Forces this object to trigger a re-index in the current state
*/
public function triggerReindex()
{
if (!$this->owner->ID) {
return;
}
$id = $this->owner->ID;
$class = $this->owner->ClassName;
$state = SearchVariant::current_state($class);
$base = ClassInfo::baseDataClass($class);
$key = "{$id}:{$base}:" . serialize($state);
$statefulids = array(array('id' => $id, 'state' => $state));
$writes = array($key => array('base' => $base, 'class' => $class, 'id' => $id, 'statefulids' => $statefulids, 'fields' => array()));
SearchUpdater::process_writes($writes);
}
/**
* Ensure the test variant is up and running properly
*/
public function testVariant()
{
// State defaults to 0
$variant = SearchVariant::current_state();
$this->assertEquals(array("SolrReindexTest_Variant" => "0"), $variant);
// All states enumerated
$allStates = iterator_to_array(SearchVariant::reindex_states());
$this->assertEquals(array(array("SolrReindexTest_Variant" => "0"), array("SolrReindexTest_Variant" => "1"), array("SolrReindexTest_Variant" => "2")), $allStates);
// Check correct items created and that filtering on variant works
$this->createDummyData(120);
SolrReindexTest_Variant::set_current(2);
$this->assertEquals(0, SolrReindexTest_Item::get()->count());
SolrReindexTest_Variant::set_current(1);
$this->assertEquals(120, SolrReindexTest_Item::get()->count());
SolrReindexTest_Variant::set_current(0);
$this->assertEquals(120, SolrReindexTest_Item::get()->count());
SolrReindexTest_Variant::disable();
$this->assertEquals(240, SolrReindexTest_Item::get()->count());
}
/**
* Generates the list of indexes to process for the dirty items
*
* @return array
*/
protected function prepareIndexes()
{
$originalState = SearchVariant::current_state();
$dirtyIndexes = array();
$dirty = $this->getSource();
$indexes = FullTextSearch::get_indexes();
foreach ($dirty as $base => $statefulids) {
if (!$statefulids) {
continue;
}
foreach ($statefulids as $statefulid) {
$state = $statefulid['state'];
$ids = $statefulid['ids'];
SearchVariant::activate_state($state);
// Ensure that indexes for all new / updated objects are included
$objs = DataObject::get($base)->byIDs(array_keys($ids));
foreach ($objs as $obj) {
foreach ($ids[$obj->ID] as $index) {
if (!$indexes[$index]->variantStateExcluded($state)) {
$indexes[$index]->add($obj);
$dirtyIndexes[$index] = $indexes[$index];
}
}
unset($ids[$obj->ID]);
}
// Generate list of records that do not exist and should be removed
foreach ($ids as $id => $fromindexes) {
foreach ($fromindexes as $index) {
if (!$indexes[$index]->variantStateExcluded($state)) {
$indexes[$index]->delete($base, $id, $state);
$dirtyIndexes[$index] = $indexes[$index];
}
}
}
}
}
SearchVariant::activate_state($originalState);
return $dirtyIndexes;
}
/**
* @deprecated since version 2.0.0
*/
protected function runFrom($index, $class, $start, $variantstate)
{
DeprecationTest_Deprecation::notice('2.0.0', 'Solr_Reindex now uses a new grouping mechanism');
// Set time limit and state
increase_time_limit_to();
SearchVariant::activate_state($variantstate);
// Generate filtered list
$items = DataList::create($class)->limit($this->config()->recordsPerRequest, $start);
// Add child filter
$classes = $index->getClasses();
$options = $classes[$class];
if (!$options['include_children']) {
$items = $items->filter('ClassName', $class);
}
// Process selected records in this class
$this->getLogger()->info("Adding {$class}");
foreach ($items->sort("ID") as $item) {
$this->getLogger()->debug($item->ID);
// See SearchUpdater_ObjectHandler::triggerReindex
$item->triggerReindex();
$item->destroy();
}
$this->getLogger()->info("Done");
}
protected function runFrom($index, $class, $start, $variantstate)
{
$classes = $index->getClasses();
$options = $classes[$class];
$verbose = isset($_GET['verbose']);
SearchVariant::activate_state($variantstate);
$includeSubclasses = $options['include_children'];
$filter = $includeSubclasses ? "" : '"ClassName" = \'' . $class . "'";
$items = DataList::create($class)->where($filter)->limit($this->stat('recordsPerRequest'), $start);
if ($verbose) {
echo "Adding {$class}";
}
foreach ($items as $item) {
if ($verbose) {
echo $item->ID . ' ';
}
// See SearchUpdater_ObjectHandler::triggerReindex
$item->triggerReindex();
$item->destroy();
}
if ($verbose) {
echo "Done ";
}
}
