/**
* Process index for a single SolrIndex instance
*
* @param LoggerInterface $logger
* @param SolrIndex $indexInstance
* @param int $batchSize
* @param string $taskName
* @param string $classes
*/
protected function processIndex(LoggerInterface $logger, SolrIndex $indexInstance, $batchSize, $taskName, $classes = null)
{
// Filter classes for this index
$indexClasses = $this->getClassesForIndex($indexInstance, $classes);
// Clear all records in this index which do not contain the given classes
$logger->info("Clearing obsolete classes from " . $indexInstance->getIndexName());
$indexInstance->clearObsoleteClasses($indexClasses);
// Build queue for each class
foreach ($indexClasses as $class => $options) {
$includeSubclasses = $options['include_children'];
foreach (SearchVariant::reindex_states($class, $includeSubclasses) as $state) {
$this->processVariant($logger, $indexInstance, $state, $class, $includeSubclasses, $batchSize, $taskName);
}
}
}
/**
* Given a class, object id, set of stateful ids and a list of changed fields (in a special format),
* return what statefulids need updating in this index
*
* Internal function used by SearchUpdater.
*
* @param $class
* @param $id
* @param $statefulids
* @param $fields
* @return array
*/
public function getDirtyIDs($class, $id, $statefulids, $fields)
{
$dirty = array();
// First, if this object is directly contained in the index, add it
foreach ($this->classes as $searchclass => $options) {
if ($searchclass == $class || $options['include_children'] && is_subclass_of($class, $searchclass)) {
$base = ClassInfo::baseDataClass($searchclass);
$dirty[$base] = array();
foreach ($statefulids as $statefulid) {
$key = serialize($statefulid);
$dirty[$base][$key] = $statefulid;
}
}
}
$current = SearchVariant::current_state();
// Then, for every derived field
foreach ($this->getDerivedFields() as $derivation) {
// If the this object is a subclass of any of the classes we want a field from
if (!SearchIntrospection::is_subclass_of($class, $derivation['classes'])) {
continue;
}
if (!array_intersect_key($fields, $derivation['fields'])) {
continue;
}
foreach (SearchVariant::reindex_states($class, false) as $state) {
SearchVariant::activate_state($state);
$ids = array($id);
foreach ($derivation['chain'] as $step) {
if ($step['through'] == 'has_one') {
$sql = new SQLQuery('"ID"', '"' . $step['class'] . '"', '"' . $step['foreignkey'] . '" IN (' . implode(',', $ids) . ')');
singleton($step['class'])->extend('augmentSQL', $sql);
$ids = $sql->execute()->column();
} else {
if ($step['through'] == 'has_many') {
$sql = new SQLQuery('"' . $step['class'] . '"."ID"', '"' . $step['class'] . '"', '"' . $step['otherclass'] . '"."ID" IN (' . implode(',', $ids) . ')');
$sql->addInnerJoin($step['otherclass'], '"' . $step['class'] . '"."ID" = "' . $step['otherclass'] . '"."' . $step['foreignkey'] . '"');
singleton($step['class'])->extend('augmentSQL', $sql);
$ids = $sql->execute()->column();
}
}
}
SearchVariant::activate_state($current);
if ($ids) {
$base = $derivation['base'];
if (!isset($dirty[$base])) {
$dirty[$base] = array();
}
foreach ($ids as $id) {
$statefulid = array('id' => $id, 'state' => $state);
$key = serialize($statefulid);
$dirty[$base][$key] = $statefulid;
}
}
}
}
return $dirty;
}
/**
* Ensure the test variant is up and running properly
*/
public function testVariant()
{
// State defaults to 0
$variant = SearchVariant::current_state();
$this->assertEquals(array("SolrReindexTest_Variant" => "0"), $variant);
// All states enumerated
$allStates = iterator_to_array(SearchVariant::reindex_states());
$this->assertEquals(array(array("SolrReindexTest_Variant" => "0"), array("SolrReindexTest_Variant" => "1"), array("SolrReindexTest_Variant" => "2")), $allStates);
// Check correct items created and that filtering on variant works
$this->createDummyData(120);
SolrReindexTest_Variant::set_current(2);
$this->assertEquals(0, SolrReindexTest_Item::get()->count());
SolrReindexTest_Variant::set_current(1);
$this->assertEquals(120, SolrReindexTest_Item::get()->count());
SolrReindexTest_Variant::set_current(0);
$this->assertEquals(120, SolrReindexTest_Item::get()->count());
SolrReindexTest_Variant::disable();
$this->assertEquals(240, SolrReindexTest_Item::get()->count());
}
public function run($request)
{
increase_time_limit_to();
$self = get_class($this);
$verbose = isset($_GET['verbose']);
$originalState = SearchVariant::current_state();
if (isset($_GET['start'])) {
$this->runFrom(singleton($_GET['index']), $_GET['class'], $_GET['start'], json_decode($_GET['variantstate'], true));
} else {
foreach (array('framework', 'sapphire') as $dirname) {
$script = sprintf("%s%s{$dirname}%scli-script.php", BASE_PATH, DIRECTORY_SEPARATOR, DIRECTORY_SEPARATOR);
if (file_exists($script)) {
break;
}
}
$class = get_class($this);
foreach (Solr::get_indexes() as $index => $instance) {
echo "Rebuilding {$instance->getIndexName()}\n\n";
$classes = $instance->getClasses();
if ($request->getVar('class')) {
$limitClasses = explode(',', $request->getVar('class'));
$classes = array_intersect_key($classes, array_combine($limitClasses, $limitClasses));
}
if ($classes) {
Solr::service($index)->deleteByQuery('ClassHierarchy:(' . implode(' OR ', array_keys($classes)) . ')');
}
foreach ($classes as $class => $options) {
$includeSubclasses = $options['include_children'];
foreach (SearchVariant::reindex_states($class, $includeSubclasses) as $state) {
if ($instance->variantStateExcluded($state)) {
continue;
}
SearchVariant::activate_state($state);
$filter = $includeSubclasses ? "" : '"ClassName" = \'' . $class . "'";
$singleton = singleton($class);
$query = $singleton->get($class, $filter, null);
$dtaQuery = $query->dataQuery();
$sqlQuery = $dtaQuery->getFinalisedQuery();
$singleton->extend('augmentSQL', $sqlQuery, $dtaQuery);
$total = $query->count();
$statevar = json_encode($state);
echo "Class: {$class}, total: {$total}";
echo $statevar ? " in state {$statevar}\n" : "\n";
if (strpos(PHP_OS, "WIN") !== false) {
$statevar = '"' . str_replace('"', '\\"', $statevar) . '"';
} else {
$statevar = "'" . $statevar . "'";
}
for ($offset = 0; $offset < $total; $offset += $this->stat('recordsPerRequest')) {
echo "{$offset}..";
$cmd = "php {$script} dev/tasks/{$self} index={$index} class={$class} start={$offset} variantstate={$statevar}";
if ($verbose) {
echo "\n Running '{$cmd}'\n";
$cmd .= " verbose=1 2>&1";
}
$res = $verbose ? passthru($cmd) : `{$cmd}`;
if ($verbose) {
echo " " . preg_replace('/\\r\\n|\\n/', '$0 ', $res) . "\n";
}
// If we're in dev mode, commit more often for fun and profit
if (Director::isDev()) {
Solr::service($index)->commit();
}
// This will slow down things a tiny bit, but it is done so that we don't timeout to the database during a reindex
DB::query('SELECT 1');
}
echo "\n";
}
}
Solr::service($index)->commit();
}
}
$originalState = SearchVariant::current_state();
}
