/**
* Returns some additional information about indexing progress, shown in
* the scheduler's task overview list.
*
* @return string Information to display
*/
public function getAdditionalInformation()
{
$message = 'Site: ' . $this->site->getLabel();
$failedItemsCount = $GLOBALS['TYPO3_DB']->exec_SELECTcountRows('uid', 'tx_solr_indexqueue_item', 'root = ' . $this->site->getRootPageId() . ' AND errors != \'\'');
if ($failedItemsCount) {
$message .= ' Failures: ' . $failedItemsCount;
}
return $message;
}
/**
* Initialize page index queue
*
* @return void
*/
protected function initializePageIndexQueue()
{
$this->pageInitializer->setIndexingConfigurationName('pages');
$this->pageInitializer->setSite(Site::getFirstAvailableSite());
$this->pageInitializer->setType('pages');
$this->pageInitializer->initialize();
}
/**
* Builds a map of indexing configuration names to tables to to index.
*
* @return array Indexing configuration to database table map
*/
protected function getIndexQueueConfigurationTableMap()
{
$indexingTableMap = array();
$solrConfiguration = $this->site->getSolrConfiguration();
$configurationNames = $solrConfiguration->getEnabledIndexQueueConfigurationNames();
foreach ($configurationNames as $configurationName) {
$indexingTableMap[$configurationName] = $solrConfiguration->getIndexQueueTableNameOrFallbackToConfigurationName($configurationName);
}
return $indexingTableMap;
}
/**
* Gets the pages in a site plus additional pages that may have been
* configured.
*
* @return array A (sorted) array of page IDs in a site
*/
protected function getPages()
{
$pages = $this->site->getPages();
$additionalPageIds = array();
if (!empty($this->indexingConfiguration['additionalPageIds'])) {
$additionalPageIds = GeneralUtility::intExplode(',', $this->indexingConfiguration['additionalPageIds']);
}
$pages = array_merge($pages, $additionalPageIds);
sort($pages, SORT_NUMERIC);
return $pages;
}
/**
* This method is designed to return some additional information about the task,
* that may help to set it apart from other tasks from the same class
* This additional information is used - for example - in the Scheduler's BE module
* This method should be implemented in most task classes
*
* @return string Information to display
*/
public function getAdditionalInformation()
{
$information = '';
if ($this->site) {
$information = 'Site: ' . $this->site->getLabel();
}
if (!empty($this->indexingConfigurationsToReIndex)) {
$information .= ', Indexing Configurations: ' . implode(', ', $this->indexingConfigurationsToReIndex);
}
return $information;
}
/**
* @return void
*/
protected function resolveSite()
{
$this->site = $this->moduleData->getSite();
if (!$this->site instanceof Site) {
$this->initializeSiteFromFirstAvailableAndStoreInModuleData();
}
$rootPageId = $this->site instanceof Site ? $this->site->getRootPageId() : 0;
if ($rootPageId > 0 && !Util::pageExists($rootPageId)) {
$this->initializeSiteFromFirstAvailableAndStoreInModuleData();
}
}
/**
* Gets the indexing progress.
*
* @return float Indexing progress as a two decimal precision float. f.e. 44.87
*/
public function getProgress()
{
$itemsIndexedPercentage = 0.0;
$totalItemsCount = $GLOBALS['TYPO3_DB']->exec_SELECTcountRows('uid', 'tx_solr_indexqueue_item', 'root = ' . $this->site->getRootPageId());
$remainingItemsCount = $GLOBALS['TYPO3_DB']->exec_SELECTcountRows('uid', 'tx_solr_indexqueue_item', 'changed > indexed AND root = ' . $this->site->getRootPageId());
$itemsIndexedCount = $totalItemsCount - $remainingItemsCount;
if ($totalItemsCount > 0) {
$itemsIndexedPercentage = $itemsIndexedCount * 100 / $totalItemsCount;
$itemsIndexedPercentage = round($itemsIndexedPercentage, 2);
}
return $itemsIndexedPercentage;
}
/**
* Checks any additional data that is relevant to this task. If the task
* class is not relevant, the method is expected to return TRUE
*
* @param array $submittedData reference to the array containing the data submitted by the user
* @param SchedulerModuleController $schedulerModule reference to the calling object (Scheduler's BE module)
* @return boolean True if validation was ok (or selected class is not relevant), FALSE otherwise
*/
public function validateAdditionalFields(array &$submittedData, SchedulerModuleController $schedulerModule)
{
$result = false;
// validate site
$sites = Site::getAvailableSites();
if (array_key_exists($submittedData['site'], $sites)) {
$result = true;
}
// escape limit
$submittedData['documentsToIndexLimit'] = intval($submittedData['documentsToIndexLimit']);
return $result;
}
protected function logInitialization($initializationQuery)
{
$solrConfiguration = $this->site->getSolrConfiguration();
$logSeverity = -1;
$logData = array('site' => $this->site->getLabel(), 'indexing configuration name' => $this->indexingConfigurationName, 'type' => $this->type, 'query' => $initializationQuery, 'rows' => $GLOBALS['TYPO3_DB']->sql_affected_rows());
if ($GLOBALS['TYPO3_DB']->sql_errno()) {
$logSeverity = 3;
$logData['error'] = $GLOBALS['TYPO3_DB']->sql_errno() . ': ' . $GLOBALS['TYPO3_DB']->sql_error();
}
if ($solrConfiguration->getLoggingIndexingIndexQueueInitialization()) {
GeneralUtility::devLog('Index Queue initialized for indexing configuration ' . $this->indexingConfigurationName, 'solr', $logSeverity, $logData);
}
}
/**
* @return mixed
*/
public function render()
{
$availableSites = Site::getAvailableSites();
$currentSite = $this->moduleDataStorageService->loadModuleData()->getSite();
$hasSites = is_array($availableSites) && count($availableSites) > 0;
$this->templateVariableContainer->add('availableSites', $availableSites);
$this->templateVariableContainer->add('currentSite', $currentSite);
$this->templateVariableContainer->add('hasSites', $hasSites);
$output = $this->renderChildren();
$this->templateVariableContainer->remove('hasSites');
$this->templateVariableContainer->remove('currentSite');
$this->templateVariableContainer->remove('availableSites');
return $output;
}
/**
* @test
*/
public function preFilledQueueContainsRootPageAfterInitialize()
{
$this->importDataSetFromFixture('can_clear_queue_after_initialize.xml');
$itemCount = $this->indexQueue->getAllItemsCount();
$this->assertItemsInQueue(1);
$this->assertFalse($this->indexQueue->containsItem('pages', 1));
$this->assertTrue($this->indexQueue->containsItem('pages', 4711));
// after initialize the prefilled queue item should be lost and the root page should be added again
$site = Site::getFirstAvailableSite();
$this->indexQueue->initialize($site, 'pages');
$this->assertItemsInQueue(1);
$this->assertTrue($this->indexQueue->containsItem('pages', 1));
$this->assertFalse($this->indexQueue->containsItem('pages', 4711));
}
public function render()
{
$this->tag->addAttribute('onchange', 'jumpToUrl(document.URL + \'&tx_solr_tools_solradministration[action]=setSite&tx_solr_tools_solradministration[site]=\'+this.options[this.selectedIndex].value,this);');
$sites = Site::getAvailableSites();
$currentSite = $this->moduleDataStorageService->loadModuleData()->getSite();
$options = '';
foreach ($sites as $site) {
$selectedAttribute = '';
if ($site == $currentSite) {
$selectedAttribute = ' selected="selected"';
}
$options .= '<option value="' . $site->getRootPageId() . '"' . $selectedAttribute . '>' . $site->getLabel() . '</option>';
}
$this->tag->setContent($options);
return '<div class="docheader-funcmenu siteSelector"><label>Site: </label>' . $this->tag->render() . '</div>';
}
/**
* Builds a map of indexing configuration names to tables to to index.
*
* @return array Indexing configuration to database table map
*/
protected function getIndexQueueConfigurationTableMap()
{
$indexingTableMap = array();
$solrConfiguration = Util::getSolrConfigurationFromPageId($this->site->getRootPageId());
foreach ($solrConfiguration['index.']['queue.'] as $name => $configuration) {
if (is_array($configuration)) {
$name = substr($name, 0, -1);
if ($solrConfiguration['index.']['queue.'][$name]) {
$table = $name;
if ($solrConfiguration['index.']['queue.'][$name . '.']['table']) {
$table = $solrConfiguration['index.']['queue.'][$name . '.']['table'];
}
$indexingTableMap[$name] = $table;
}
}
}
return $indexingTableMap;
}
/**
* Retrieves the value that should be used for the SiteHash filter.
*
* @param TypoScriptFrontendController $TSFE
* @return string
*/
protected function getSiteHashFilterForTSFE(TypoScriptFrontendController $TSFE)
{
return Site::getSiteByPageId($TSFE->id)->getDomain();
}
/**
* Resolves magic keywords in allowed sites configuration.
* Supported keywords:
* __solr_current_site - The domain of the site the query has been started from
* __current_site - Same as __solr_current_site
* __all - Adds all domains as allowed sites
* * - Same as __all
*
* @param integer $pageId A page ID that is then resolved to the site it belongs to
* @param string $allowedSitesConfiguration TypoScript setting for allowed sites
* @return string List of allowed sites/domains, magic keywords resolved
*/
public static function resolveSiteHashAllowedSites($pageId, $allowedSitesConfiguration)
{
if ($allowedSitesConfiguration == '*' || $allowedSitesConfiguration == '__all') {
$sites = Site::getAvailableSites();
$domains = array();
foreach ($sites as $site) {
$domains[] = $site->getDomain();
}
$allowedSites = implode(',', $domains);
} else {
$allowedSites = str_replace(array('__solr_current_site', '__current_site'), Site::getSiteByPageId($pageId)->getDomain(), $allowedSitesConfiguration);
}
return $allowedSites;
}
/**
* Resets the stored site to the first available site.
*
* @return void‚
*/
protected function initializeSiteFromFirstAvailableAndStoreInModuleData()
{
$site = Site::getFirstAvailableSite();
$this->setSiteAndResetCore($site);
$this->site = $site;
}
/**
* Builds the Solr document for the current page.
*
* @return \Apache_Solr_Document A document representing the page
*/
protected function getPageDocument()
{
$document = GeneralUtility::makeInstance('\\Apache_Solr_Document');
/* @var $document \Apache_Solr_Document */
$site = Site::getSiteByPageId($this->page->id);
$pageRecord = $this->page->page;
self::$pageSolrDocumentId = $documentId = Util::getPageDocumentId($this->page->id, $this->page->type, $this->page->sys_language_uid, $this->getDocumentIdGroups());
$document->setField('id', $documentId);
$document->setField('site', $site->getDomain());
$document->setField('siteHash', $site->getSiteHash());
$document->setField('appKey', 'EXT:solr');
$document->setField('type', 'pages');
// system fields
$document->setField('uid', $this->page->id);
$document->setField('pid', $pageRecord['pid']);
$document->setField('typeNum', $this->page->type);
$document->setField('created', $pageRecord['crdate']);
$document->setField('changed', $pageRecord['tstamp']);
$document->setField('rootline', $this->page->id);
// access
$document->setField('access', (string) $this->pageAccessRootline);
if ($this->page->page['endtime']) {
$document->setField('endtime', $pageRecord['endtime']);
}
// content
$document->setField('title', $this->contentExtractor->getPageTitle());
$document->setField('subTitle', $pageRecord['subtitle']);
$document->setField('navTitle', $pageRecord['nav_title']);
$document->setField('author', $pageRecord['author']);
$document->setField('description', $pageRecord['description']);
$document->setField('abstract', $pageRecord['abstract']);
$document->setField('content', $this->contentExtractor->getIndexableContent());
$document->setField('url', $this->pageUrl);
// keywords, multi valued
$keywords = array_unique(GeneralUtility::trimExplode(',', $pageRecord['keywords'], TRUE));
foreach ($keywords as $keyword) {
$document->addField('keywords', $keyword);
}
// content from several tags like headers, anchors, ...
$tagContent = $this->contentExtractor->getTagContent();
foreach ($tagContent as $fieldName => $fieldValue) {
$document->setField($fieldName, $fieldValue);
}
return $document;
}
/**
* @test
*/
public function canGetAdditionalInformationFromTask()
{
$this->importDataSetFromFixture('can_trigger_frontend_calls_for_page_index.xml');
$site = Site::getFirstAvailableSite();
/** @var $indexQueueQueueWorkerTask \ApacheSolrForTypo3\Solr\Task\IndexQueueWorkerTask */
$indexQueueQueueWorkerTask = GeneralUtility::makeInstance('ApacheSolrForTypo3\\Solr\\Task\\IndexQueueWorkerTask');
$indexQueueQueueWorkerTask->setDocumentsToIndexLimit(1);
$indexQueueQueueWorkerTask->setSite($site);
$additionalInformation = $indexQueueQueueWorkerTask->getAdditionalInformation();
$this->assertContains('Root Page ID: 1', $additionalInformation);
$this->assertContains('Site: page for testing', $additionalInformation);
}
/**
* @test
*/
public function canGetDefaultLanguage()
{
$this->importDataSetFromFixture('can_get_default_language.xml');
$site = Site::getFirstAvailableSite();
$this->assertEquals(888, $site->getDefaultLanguage(), 'Could not get default language from site');
}
/**
* Gets all the pages from a mounted page tree.
*
* @param integer $mountPageSourceId
* @return array An array of page IDs in the mounted page tree
*/
protected function resolveMountPageTree($mountPageSourceId)
{
$mountedSite = Site::getSiteByPageId($mountPageSourceId);
return $mountedSite->getPages($mountPageSourceId);
}
/**
* Queries Solr for the current page's documents.
*
* @return array An array of Apache_Solr_Document objects
*/
protected function getIndexDocuments()
{
/* @var Query $query */
$query = GeneralUtility::makeInstance('ApacheSolrForTypo3\\Solr\\Query', '');
$query->setQueryType('standard');
$query->useRawQueryString(true);
$query->setQueryString('*:*');
$query->addFilter('(type:pages AND uid:' . $this->pageId . ') OR (*:* AND pid:' . $this->pageId . ' NOT type:pages)');
$query->addFilter('siteHash:' . Site::getSiteByPageId($this->pageId)->getSiteHash());
$query->setFieldList('*');
$query->setSorting('type asc, title asc');
$this->search->search($query, 0, 10000);
return $this->search->getResultDocumentsEscaped();
}
/**
* Gets all connection configurations for a given site.
*
* @param Site $site A TYPO3 site
* @return array An array of Solr connection configurations for a site
*/
public function getConfigurationsBySite(Site $site)
{
$solrConfigurations = array();
$allConfigurations = $this->getAllConfigurations();
foreach ($allConfigurations as $configuration) {
if ($configuration['rootPageUid'] == $site->getRootPageId()) {
$solrConfigurations[] = $configuration;
}
}
return $solrConfigurations;
}
/**
*
* @dataProvider canResolveAbsRefPrefixDataProvider
* @param string $absRefPrefix
* @param string $expectedUrl
* @test
*/
public function canResolveAbsRefPrefix($absRefPrefix, $expectedUrl)
{
$this->cleanUpSolrServerAndAssertEmpty();
// create fake extension database table and TCA
$this->importDumpFromFixture('fake_extension2_table.sql');
$GLOBALS['TCA']['tx_fakeextension_domain_model_bar'] = (include $this->getFixturePath('fake_extension2_bar_tca.php'));
$GLOBALS['TCA']['tx_fakeextension_domain_model_directrelated'] = (include $this->getFixturePath('fake_extension2_directrelated_tca.php'));
$this->importDataSetFromFixture('can_index_custom_record_absRefPrefix_' . $absRefPrefix . '.xml');
$this->addToIndexQueue('tx_fakeextension_domain_model_bar', 111);
/** @var $cliEnvironment CliEnvironment */
$cliEnvironment = GeneralUtility::makeInstance(CliEnvironment::class);
$cliEnvironment->backup();
$cliEnvironment->initialize(PATH_site);
/** @var $indexService IndexService */
$site = Site::getFirstAvailableSite();
$indexService = GeneralUtility::makeInstance(IndexService::class, $site);
// run the indexer
$indexService->indexItems(1);
$cliEnvironment->restore();
// do we have the record in the index with the value from the mm relation?
sleep(2);
$solrContent = file_get_contents('http://localhost:8080/solr/core_en/select?q=*:*');
$this->assertContains('"numFound":1', $solrContent, 'Could not index document into solr');
$this->assertContains('"url":"' . $expectedUrl . '"', $solrContent, 'Generated unexpected url with absRefPrefix = auto');
$this->assertNotContains('auto', $solrContent, 'absRefPrefix=auto was not resolved');
$this->cleanUpSolrServerAndAssertEmpty();
}
/**
* Gets $limit number of items to index for a particular $site.
*
* @param Site $site TYPO3 site
* @param integer $limit Number of items to get from the queue
* @return Item[] Items to index to the given solr server
*/
public function getItemsToIndex(Site $site, $limit = 50)
{
$itemsToIndex = array();
// determine which items to index with this run
$indexQueueItemRecords = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows('*', 'tx_solr_indexqueue_item', 'root = ' . $site->getRootPageId() . ' AND changed > indexed' . ' AND changed <= ' . time() . ' AND errors = \'\'', '', 'indexing_priority DESC, changed DESC, uid DESC', intval($limit));
if (!empty($indexQueueItemRecords)) {
// convert queued records to index queue item objects
$itemsToIndex = $this->getIndexQueueItemObjectsFromRecords($indexQueueItemRecords);
}
return $itemsToIndex;
}
/**
* Checks any additional data that is relevant to this task. If the task
* class is not relevant, the method is expected to return TRUE
*
* @param array $submittedData reference to the array containing the data submitted by the user
* @param SchedulerModuleController $schedulerModule reference to the calling object (Scheduler's BE module)
* @return bool True if validation was ok (or selected class is not relevant), FALSE otherwise
*/
public function validateAdditionalFields(array &$submittedData, SchedulerModuleController $schedulerModule)
{
$result = false;
// validate site
$sites = Site::getAvailableSites();
if (array_key_exists($submittedData['site'], $sites)) {
$result = true;
}
return $result;
}
/**
* Initializes resources commonly needed for several actions
*
* @return void
*/
protected function initializeAction()
{
try {
$site = $this->request->getArgument('site');
if (is_numeric($site)) {
$siteRootPageId = $this->request->getArgument('site');
$this->site = Site::getSiteByPageId($siteRootPageId);
} else {
if ($site instanceof Site) {
$this->site = $site;
}
}
} catch (NoSuchArgumentException $nsae) {
$sites = Site::getAvailableSites();
$site = array_shift($sites);
$this->site = $site;
}
$this->request->setArgument('site', $this->site);
$moduleData = $this->moduleDataStorageService->loadModuleData();
$moduleData->setSite($this->site);
$this->moduleDataStorageService->persistModuleData($moduleData);
}
/**
* @test
*/
public function canGetAllSites()
{
$this->importDataSetFromFixture('can_get_all_sites.xml');
$sites = Site::getAvailableSites();
$this->assertSame(1, count($sites), 'Expected to retrieve one site from fixture');
}
/**
* @test
*/
public function solrIsEmptyAfterCleanup()
{
$this->importDataSetFromFixture('can_reindex_task_fill_queue.xml');
// fill the solr
$site = Site::getFirstAvailableSite();
$this->indexQueue->updateItem('pages', 1);
$items = $this->indexQueue->getItems('pages', 1);
/** @var $indexer \ApacheSolrForTypo3\Solr\IndexQueue\Indexer */
$indexer = GeneralUtility::makeInstance('ApacheSolrForTypo3\\Solr\\IndexQueue\\Indexer');
$indexer->index($items[0]);
sleep(2);
$this->assertSolrContainsDocumentCount(1);
$this->task->setSite($site);
$this->task->setIndexingConfigurationsToReIndex(array('pages'));
$this->task->execute();
sleep(2);
// after the task was running the solr server should be empty
$this->assertSolrIsEmpty();
// if not we cleanup now
$this->cleanUpSolrServerAndAssertEmpty();
}
/**
* Adds a page to the Index Queue of a site mounting the page.
*
* @param integer $mountedPageId ID (uid) of the mounted page.
* @param array $mountProperties Array of mount point properties mountPageSource, mountPageDestination, and mountPageOverlayed
*/
protected function addPageToMountingSiteIndexQueue($mountedPageId, array $mountProperties)
{
$mountingSite = Site::getSiteByPageId($mountProperties['mountPageDestination']);
$pageInitializer = GeneralUtility::makeInstance('ApacheSolrForTypo3\\Solr\\IndexQueue\\Initializer\\Page');
$pageInitializer->setSite($mountingSite);
$pageInitializer->initializeMountedPage($mountProperties, $mountedPageId);
}
/**
* Finds the alternative page language overlay records for a page based on
* the sys_language_mode.
*
* Possible Language Modes:
* 1) content_fallback --> all languages
* 2) strict --> available languages with page overlay
* 3) ignore --> available languages with page overlay
* 4) unknown mode or blank --> all languages
*
* @param integer $pageId Page ID.
* @return array An array of translation overlays (or fake overlays) found for the given page.
*/
protected function getTranslationOverlaysForPage($pageId)
{
$translationOverlays = array();
$pageId = intval($pageId);
$site = Site::getSiteByPageId($pageId);
$languageModes = array('content_fallback', 'strict', 'ignore');
$hasOverlayMode = in_array($site->getSysLanguageMode(), $languageModes, TRUE);
$isContentFallbackMode = $site->getSysLanguageMode() === 'content_fallback';
if ($hasOverlayMode && !$isContentFallbackMode) {
$translationOverlays = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows('pid, sys_language_uid', 'pages_language_overlay', 'pid = ' . $pageId . BackendUtility::deleteClause('pages_language_overlay') . BackendUtility::BEenableFields('pages_language_overlay'));
} else {
// ! If no sys_language_mode is configured, all languages will be indexed !
$languages = BackendUtility::getSystemLanguages();
// remove default language (L = 0)
array_shift($languages);
foreach ($languages as $language) {
$translationOverlays[] = array('pid' => $pageId, 'sys_language_uid' => $language[1]);
}
}
return $translationOverlays;
}
