/** * Generates the list of indexes to process for the dirty items * * @return array */ protected function prepareIndexes() { $originalState = SearchVariant::current_state(); $dirtyIndexes = array(); $dirty = $this->getSource(); $indexes = FullTextSearch::get_indexes(); foreach ($dirty as $base => $statefulids) { if (!$statefulids) { continue; } foreach ($statefulids as $statefulid) { $state = $statefulid['state']; $ids = $statefulid['ids']; SearchVariant::activate_state($state); // Ensure that indexes for all new / updated objects are included $objs = DataObject::get($base)->byIDs(array_keys($ids)); foreach ($objs as $obj) { foreach ($ids[$obj->ID] as $index) { if (!$indexes[$index]->variantStateExcluded($state)) { $indexes[$index]->add($obj); $dirtyIndexes[$index] = $indexes[$index]; } } unset($ids[$obj->ID]); } // Generate list of records that do not exist and should be removed foreach ($ids as $id => $fromindexes) { foreach ($fromindexes as $index) { if (!$indexes[$index]->variantStateExcluded($state)) { $indexes[$index]->delete($base, $id, $state); $dirtyIndexes[$index] = $indexes[$index]; } } } } } SearchVariant::activate_state($originalState); return $dirtyIndexes; }
/** * Given a class, object id, set of stateful ids and a list of changed fields (in a special format), * return what statefulids need updating in this index * * Internal function used by SearchUpdater. * * @param $class * @param $id * @param $statefulids * @param $fields * @return array */ public function getDirtyIDs($class, $id, $statefulids, $fields) { $dirty = array(); // First, if this object is directly contained in the index, add it foreach ($this->classes as $searchclass => $options) { if ($searchclass == $class || $options['include_children'] && is_subclass_of($class, $searchclass)) { $base = ClassInfo::baseDataClass($searchclass); $dirty[$base] = array(); foreach ($statefulids as $statefulid) { $key = serialize($statefulid); $dirty[$base][$key] = $statefulid; } } } $current = SearchVariant::current_state(); // Then, for every derived field foreach ($this->getDerivedFields() as $derivation) { // If the this object is a subclass of any of the classes we want a field from if (!SearchIntrospection::is_subclass_of($class, $derivation['classes'])) { continue; } if (!array_intersect_key($fields, $derivation['fields'])) { continue; } foreach (SearchVariant::reindex_states($class, false) as $state) { SearchVariant::activate_state($state); $ids = array($id); foreach ($derivation['chain'] as $step) { if ($step['through'] == 'has_one') { $sql = new SQLQuery('"ID"', '"' . $step['class'] . '"', '"' . $step['foreignkey'] . '" IN (' . implode(',', $ids) . ')'); singleton($step['class'])->extend('augmentSQL', $sql); $ids = $sql->execute()->column(); } else { if ($step['through'] == 'has_many') { $sql = new SQLQuery('"' . $step['class'] . '"."ID"', '"' . $step['class'] . '"', '"' . $step['otherclass'] . '"."ID" IN (' . implode(',', $ids) . ')'); $sql->addInnerJoin($step['otherclass'], '"' . $step['class'] . '"."ID" = "' . $step['otherclass'] . '"."' . $step['foreignkey'] . '"'); singleton($step['class'])->extend('augmentSQL', $sql); $ids = $sql->execute()->column(); } } } SearchVariant::activate_state($current); if ($ids) { $base = $derivation['base']; if (!isset($dirty[$base])) { $dirty[$base] = array(); } foreach ($ids as $id) { $statefulid = array('id' => $id, 'state' => $state); $key = serialize($statefulid); $dirty[$base][$key] = $statefulid; } } } } return $dirty; }
/** * Forces this object to trigger a re-index in the current state */ public function triggerReindex() { if (!$this->owner->ID) { return; } $id = $this->owner->ID; $class = $this->owner->ClassName; $state = SearchVariant::current_state($class); $base = ClassInfo::baseDataClass($class); $key = "{$id}:{$base}:" . serialize($state); $statefulids = array(array('id' => $id, 'state' => $state)); $writes = array($key => array('base' => $base, 'class' => $class, 'id' => $id, 'statefulids' => $statefulids, 'fields' => array())); SearchUpdater::process_writes($writes); }
/** * Ensure the test variant is up and running properly */ public function testVariant() { // State defaults to 0 $variant = SearchVariant::current_state(); $this->assertEquals(array("SolrReindexTest_Variant" => "0"), $variant); // All states enumerated $allStates = iterator_to_array(SearchVariant::reindex_states()); $this->assertEquals(array(array("SolrReindexTest_Variant" => "0"), array("SolrReindexTest_Variant" => "1"), array("SolrReindexTest_Variant" => "2")), $allStates); // Check correct items created and that filtering on variant works $this->createDummyData(120); SolrReindexTest_Variant::set_current(2); $this->assertEquals(0, SolrReindexTest_Item::get()->count()); SolrReindexTest_Variant::set_current(1); $this->assertEquals(120, SolrReindexTest_Item::get()->count()); SolrReindexTest_Variant::set_current(0); $this->assertEquals(120, SolrReindexTest_Item::get()->count()); SolrReindexTest_Variant::disable(); $this->assertEquals(240, SolrReindexTest_Item::get()->count()); }
/** * @param SS_HTTPRequest $request */ public function run($request) { parent::run($request); // Reset state $originalState = SearchVariant::current_state(); $this->doReindex($request); SearchVariant::activate_state($originalState); }
public function run($request) { increase_time_limit_to(); $self = get_class($this); $verbose = isset($_GET['verbose']); $originalState = SearchVariant::current_state(); if (isset($_GET['start'])) { $this->runFrom(singleton($_GET['index']), $_GET['class'], $_GET['start'], json_decode($_GET['variantstate'], true)); } else { foreach (array('framework', 'sapphire') as $dirname) { $script = sprintf("%s%s{$dirname}%scli-script.php", BASE_PATH, DIRECTORY_SEPARATOR, DIRECTORY_SEPARATOR); if (file_exists($script)) { break; } } $class = get_class($this); foreach (Solr::get_indexes() as $index => $instance) { echo "Rebuilding {$instance->getIndexName()}\n\n"; $classes = $instance->getClasses(); if ($request->getVar('class')) { $limitClasses = explode(',', $request->getVar('class')); $classes = array_intersect_key($classes, array_combine($limitClasses, $limitClasses)); } if ($classes) { Solr::service($index)->deleteByQuery('ClassHierarchy:(' . implode(' OR ', array_keys($classes)) . ')'); } foreach ($classes as $class => $options) { $includeSubclasses = $options['include_children']; foreach (SearchVariant::reindex_states($class, $includeSubclasses) as $state) { if ($instance->variantStateExcluded($state)) { continue; } SearchVariant::activate_state($state); $filter = $includeSubclasses ? "" : '"ClassName" = \'' . $class . "'"; $singleton = singleton($class); $query = $singleton->get($class, $filter, null); $dtaQuery = $query->dataQuery(); $sqlQuery = $dtaQuery->getFinalisedQuery(); $singleton->extend('augmentSQL', $sqlQuery, $dtaQuery); $total = $query->count(); $statevar = json_encode($state); echo "Class: {$class}, total: {$total}"; echo $statevar ? " in state {$statevar}\n" : "\n"; if (strpos(PHP_OS, "WIN") !== false) { $statevar = '"' . str_replace('"', '\\"', $statevar) . '"'; } else { $statevar = "'" . $statevar . "'"; } for ($offset = 0; $offset < $total; $offset += $this->stat('recordsPerRequest')) { echo "{$offset}.."; $cmd = "php {$script} dev/tasks/{$self} index={$index} class={$class} start={$offset} variantstate={$statevar}"; if ($verbose) { echo "\n Running '{$cmd}'\n"; $cmd .= " verbose=1 2>&1"; } $res = $verbose ? passthru($cmd) : `{$cmd}`; if ($verbose) { echo " " . preg_replace('/\\r\\n|\\n/', '$0 ', $res) . "\n"; } // If we're in dev mode, commit more often for fun and profit if (Director::isDev()) { Solr::service($index)->commit(); } // This will slow down things a tiny bit, but it is done so that we don't timeout to the database during a reindex DB::query('SELECT 1'); } echo "\n"; } } Solr::service($index)->commit(); } } $originalState = SearchVariant::current_state(); }