Esempio n. 1
0
 /**
  * Returns some additional information about indexing progress, shown in
  * the scheduler's task overview list.
  *
  * @return string Information to display
  */
 public function getAdditionalInformation()
 {
     $message = 'Site: ' . $this->site->getLabel();
     $failedItemsCount = $GLOBALS['TYPO3_DB']->exec_SELECTcountRows('uid', 'tx_solr_indexqueue_item', 'root = ' . $this->site->getRootPageId() . ' AND errors != \'\'');
     if ($failedItemsCount) {
         $message .= ' Failures: ' . $failedItemsCount;
     }
     return $message;
 }
Esempio n. 2
0
 /**
  * Initialize page index queue
  *
  * @return void
  */
 protected function initializePageIndexQueue()
 {
     $this->pageInitializer->setIndexingConfigurationName('pages');
     $this->pageInitializer->setSite(Site::getFirstAvailableSite());
     $this->pageInitializer->setType('pages');
     $this->pageInitializer->initialize();
 }
 /**
  * Builds a map of indexing configuration names to tables to to index.
  *
  * @return array Indexing configuration to database table map
  */
 protected function getIndexQueueConfigurationTableMap()
 {
     $indexingTableMap = array();
     $solrConfiguration = $this->site->getSolrConfiguration();
     $configurationNames = $solrConfiguration->getEnabledIndexQueueConfigurationNames();
     foreach ($configurationNames as $configurationName) {
         $indexingTableMap[$configurationName] = $solrConfiguration->getIndexQueueTableNameOrFallbackToConfigurationName($configurationName);
     }
     return $indexingTableMap;
 }
Esempio n. 4
0
 /**
  * Gets the pages in a site plus additional pages that may have been
  * configured.
  *
  * @return array A (sorted) array of page IDs in a site
  */
 protected function getPages()
 {
     $pages = $this->site->getPages();
     $additionalPageIds = array();
     if (!empty($this->indexingConfiguration['additionalPageIds'])) {
         $additionalPageIds = GeneralUtility::intExplode(',', $this->indexingConfiguration['additionalPageIds']);
     }
     $pages = array_merge($pages, $additionalPageIds);
     sort($pages, SORT_NUMERIC);
     return $pages;
 }
Esempio n. 5
0
 /**
  * This method is designed to return some additional information about the task,
  * that may help to set it apart from other tasks from the same class
  * This additional information is used - for example - in the Scheduler's BE module
  * This method should be implemented in most task classes
  *
  * @return string Information to display
  */
 public function getAdditionalInformation()
 {
     $information = '';
     if ($this->site) {
         $information = 'Site: ' . $this->site->getLabel();
     }
     if (!empty($this->indexingConfigurationsToReIndex)) {
         $information .= ', Indexing Configurations: ' . implode(', ', $this->indexingConfigurationsToReIndex);
     }
     return $information;
 }
 /**
  * @return void
  */
 protected function resolveSite()
 {
     $this->site = $this->moduleData->getSite();
     if (!$this->site instanceof Site) {
         $this->initializeSiteFromFirstAvailableAndStoreInModuleData();
     }
     $rootPageId = $this->site instanceof Site ? $this->site->getRootPageId() : 0;
     if ($rootPageId > 0 && !Util::pageExists($rootPageId)) {
         $this->initializeSiteFromFirstAvailableAndStoreInModuleData();
     }
 }
Esempio n. 7
0
 /**
  * Gets the indexing progress.
  *
  * @return float Indexing progress as a two decimal precision float. f.e. 44.87
  */
 public function getProgress()
 {
     $itemsIndexedPercentage = 0.0;
     $totalItemsCount = $GLOBALS['TYPO3_DB']->exec_SELECTcountRows('uid', 'tx_solr_indexqueue_item', 'root = ' . $this->site->getRootPageId());
     $remainingItemsCount = $GLOBALS['TYPO3_DB']->exec_SELECTcountRows('uid', 'tx_solr_indexqueue_item', 'changed > indexed AND root = ' . $this->site->getRootPageId());
     $itemsIndexedCount = $totalItemsCount - $remainingItemsCount;
     if ($totalItemsCount > 0) {
         $itemsIndexedPercentage = $itemsIndexedCount * 100 / $totalItemsCount;
         $itemsIndexedPercentage = round($itemsIndexedPercentage, 2);
     }
     return $itemsIndexedPercentage;
 }
 /**
  * Checks any additional data that is relevant to this task. If the task
  * class is not relevant, the method is expected to return TRUE
  *
  * @param array $submittedData reference to the array containing the data submitted by the user
  * @param SchedulerModuleController $schedulerModule reference to the calling object (Scheduler's BE module)
  * @return boolean True if validation was ok (or selected class is not relevant), FALSE otherwise
  */
 public function validateAdditionalFields(array &$submittedData, SchedulerModuleController $schedulerModule)
 {
     $result = false;
     // validate site
     $sites = Site::getAvailableSites();
     if (array_key_exists($submittedData['site'], $sites)) {
         $result = true;
     }
     // escape limit
     $submittedData['documentsToIndexLimit'] = intval($submittedData['documentsToIndexLimit']);
     return $result;
 }
Esempio n. 9
0
 protected function logInitialization($initializationQuery)
 {
     $solrConfiguration = $this->site->getSolrConfiguration();
     $logSeverity = -1;
     $logData = array('site' => $this->site->getLabel(), 'indexing configuration name' => $this->indexingConfigurationName, 'type' => $this->type, 'query' => $initializationQuery, 'rows' => $GLOBALS['TYPO3_DB']->sql_affected_rows());
     if ($GLOBALS['TYPO3_DB']->sql_errno()) {
         $logSeverity = 3;
         $logData['error'] = $GLOBALS['TYPO3_DB']->sql_errno() . ': ' . $GLOBALS['TYPO3_DB']->sql_error();
     }
     if ($solrConfiguration->getLoggingIndexingIndexQueueInitialization()) {
         GeneralUtility::devLog('Index Queue initialized for indexing configuration ' . $this->indexingConfigurationName, 'solr', $logSeverity, $logData);
     }
 }
Esempio n. 10
0
 /**
  * @return mixed
  */
 public function render()
 {
     $availableSites = Site::getAvailableSites();
     $currentSite = $this->moduleDataStorageService->loadModuleData()->getSite();
     $hasSites = is_array($availableSites) && count($availableSites) > 0;
     $this->templateVariableContainer->add('availableSites', $availableSites);
     $this->templateVariableContainer->add('currentSite', $currentSite);
     $this->templateVariableContainer->add('hasSites', $hasSites);
     $output = $this->renderChildren();
     $this->templateVariableContainer->remove('hasSites');
     $this->templateVariableContainer->remove('currentSite');
     $this->templateVariableContainer->remove('availableSites');
     return $output;
 }
Esempio n. 11
0
 /**
  * @test
  */
 public function preFilledQueueContainsRootPageAfterInitialize()
 {
     $this->importDataSetFromFixture('can_clear_queue_after_initialize.xml');
     $itemCount = $this->indexQueue->getAllItemsCount();
     $this->assertItemsInQueue(1);
     $this->assertFalse($this->indexQueue->containsItem('pages', 1));
     $this->assertTrue($this->indexQueue->containsItem('pages', 4711));
     // after initialize the prefilled queue item should be lost and the root page should be added again
     $site = Site::getFirstAvailableSite();
     $this->indexQueue->initialize($site, 'pages');
     $this->assertItemsInQueue(1);
     $this->assertTrue($this->indexQueue->containsItem('pages', 1));
     $this->assertFalse($this->indexQueue->containsItem('pages', 4711));
 }
 public function render()
 {
     $this->tag->addAttribute('onchange', 'jumpToUrl(document.URL + \'&tx_solr_tools_solradministration[action]=setSite&tx_solr_tools_solradministration[site]=\'+this.options[this.selectedIndex].value,this);');
     $sites = Site::getAvailableSites();
     $currentSite = $this->moduleDataStorageService->loadModuleData()->getSite();
     $options = '';
     foreach ($sites as $site) {
         $selectedAttribute = '';
         if ($site == $currentSite) {
             $selectedAttribute = ' selected="selected"';
         }
         $options .= '<option value="' . $site->getRootPageId() . '"' . $selectedAttribute . '>' . $site->getLabel() . '</option>';
     }
     $this->tag->setContent($options);
     return '<div class="docheader-funcmenu siteSelector"><label>Site: </label>' . $this->tag->render() . '</div>';
 }
 /**
  * Builds a map of indexing configuration names to tables to to index.
  *
  * @return array Indexing configuration to database table map
  */
 protected function getIndexQueueConfigurationTableMap()
 {
     $indexingTableMap = array();
     $solrConfiguration = Util::getSolrConfigurationFromPageId($this->site->getRootPageId());
     foreach ($solrConfiguration['index.']['queue.'] as $name => $configuration) {
         if (is_array($configuration)) {
             $name = substr($name, 0, -1);
             if ($solrConfiguration['index.']['queue.'][$name]) {
                 $table = $name;
                 if ($solrConfiguration['index.']['queue.'][$name . '.']['table']) {
                     $table = $solrConfiguration['index.']['queue.'][$name . '.']['table'];
                 }
                 $indexingTableMap[$name] = $table;
             }
         }
     }
     return $indexingTableMap;
 }
Esempio n. 14
0
 /**
  * Retrieves the value that should be used for the SiteHash filter.
  *
  * @param TypoScriptFrontendController $TSFE
  * @return string
  */
 protected function getSiteHashFilterForTSFE(TypoScriptFrontendController $TSFE)
 {
     return Site::getSiteByPageId($TSFE->id)->getDomain();
 }
Esempio n. 15
0
 /**
  * Resolves magic keywords in allowed sites configuration.
  * Supported keywords:
  *   __solr_current_site - The domain of the site the query has been started from
  *   __current_site - Same as __solr_current_site
  *   __all - Adds all domains as allowed sites
  *   * - Same as __all
  *
  * @param integer $pageId A page ID that is then resolved to the site it belongs to
  * @param string $allowedSitesConfiguration TypoScript setting for allowed sites
  * @return string List of allowed sites/domains, magic keywords resolved
  */
 public static function resolveSiteHashAllowedSites($pageId, $allowedSitesConfiguration)
 {
     if ($allowedSitesConfiguration == '*' || $allowedSitesConfiguration == '__all') {
         $sites = Site::getAvailableSites();
         $domains = array();
         foreach ($sites as $site) {
             $domains[] = $site->getDomain();
         }
         $allowedSites = implode(',', $domains);
     } else {
         $allowedSites = str_replace(array('__solr_current_site', '__current_site'), Site::getSiteByPageId($pageId)->getDomain(), $allowedSitesConfiguration);
     }
     return $allowedSites;
 }
 /**
  * Resets the stored site to the first available site.
  *
  * @return void‚
  */
 protected function initializeSiteFromFirstAvailableAndStoreInModuleData()
 {
     $site = Site::getFirstAvailableSite();
     $this->setSiteAndResetCore($site);
     $this->site = $site;
 }
Esempio n. 17
0
 /**
  * Builds the Solr document for the current page.
  *
  * @return \Apache_Solr_Document A document representing the page
  */
 protected function getPageDocument()
 {
     $document = GeneralUtility::makeInstance('\\Apache_Solr_Document');
     /* @var $document \Apache_Solr_Document */
     $site = Site::getSiteByPageId($this->page->id);
     $pageRecord = $this->page->page;
     self::$pageSolrDocumentId = $documentId = Util::getPageDocumentId($this->page->id, $this->page->type, $this->page->sys_language_uid, $this->getDocumentIdGroups());
     $document->setField('id', $documentId);
     $document->setField('site', $site->getDomain());
     $document->setField('siteHash', $site->getSiteHash());
     $document->setField('appKey', 'EXT:solr');
     $document->setField('type', 'pages');
     // system fields
     $document->setField('uid', $this->page->id);
     $document->setField('pid', $pageRecord['pid']);
     $document->setField('typeNum', $this->page->type);
     $document->setField('created', $pageRecord['crdate']);
     $document->setField('changed', $pageRecord['tstamp']);
     $document->setField('rootline', $this->page->id);
     // access
     $document->setField('access', (string) $this->pageAccessRootline);
     if ($this->page->page['endtime']) {
         $document->setField('endtime', $pageRecord['endtime']);
     }
     // content
     $document->setField('title', $this->contentExtractor->getPageTitle());
     $document->setField('subTitle', $pageRecord['subtitle']);
     $document->setField('navTitle', $pageRecord['nav_title']);
     $document->setField('author', $pageRecord['author']);
     $document->setField('description', $pageRecord['description']);
     $document->setField('abstract', $pageRecord['abstract']);
     $document->setField('content', $this->contentExtractor->getIndexableContent());
     $document->setField('url', $this->pageUrl);
     // keywords, multi valued
     $keywords = array_unique(GeneralUtility::trimExplode(',', $pageRecord['keywords'], TRUE));
     foreach ($keywords as $keyword) {
         $document->addField('keywords', $keyword);
     }
     // content from several tags like headers, anchors, ...
     $tagContent = $this->contentExtractor->getTagContent();
     foreach ($tagContent as $fieldName => $fieldValue) {
         $document->setField($fieldName, $fieldValue);
     }
     return $document;
 }
 /**
  * @test
  */
 public function canGetAdditionalInformationFromTask()
 {
     $this->importDataSetFromFixture('can_trigger_frontend_calls_for_page_index.xml');
     $site = Site::getFirstAvailableSite();
     /** @var $indexQueueQueueWorkerTask \ApacheSolrForTypo3\Solr\Task\IndexQueueWorkerTask */
     $indexQueueQueueWorkerTask = GeneralUtility::makeInstance('ApacheSolrForTypo3\\Solr\\Task\\IndexQueueWorkerTask');
     $indexQueueQueueWorkerTask->setDocumentsToIndexLimit(1);
     $indexQueueQueueWorkerTask->setSite($site);
     $additionalInformation = $indexQueueQueueWorkerTask->getAdditionalInformation();
     $this->assertContains('Root Page ID: 1', $additionalInformation);
     $this->assertContains('Site: page for testing', $additionalInformation);
 }
Esempio n. 19
0
 /**
  * @test
  */
 public function canGetDefaultLanguage()
 {
     $this->importDataSetFromFixture('can_get_default_language.xml');
     $site = Site::getFirstAvailableSite();
     $this->assertEquals(888, $site->getDefaultLanguage(), 'Could not get default language from site');
 }
Esempio n. 20
0
 /**
  * Gets all the pages from a mounted page tree.
  *
  * @param integer $mountPageSourceId
  * @return array An array of page IDs in the mounted page tree
  */
 protected function resolveMountPageTree($mountPageSourceId)
 {
     $mountedSite = Site::getSiteByPageId($mountPageSourceId);
     return $mountedSite->getPages($mountPageSourceId);
 }
 /**
  * Queries Solr for the current page's documents.
  *
  * @return array An array of Apache_Solr_Document objects
  */
 protected function getIndexDocuments()
 {
     /* @var Query $query */
     $query = GeneralUtility::makeInstance('ApacheSolrForTypo3\\Solr\\Query', '');
     $query->setQueryType('standard');
     $query->useRawQueryString(true);
     $query->setQueryString('*:*');
     $query->addFilter('(type:pages AND uid:' . $this->pageId . ') OR (*:* AND pid:' . $this->pageId . ' NOT type:pages)');
     $query->addFilter('siteHash:' . Site::getSiteByPageId($this->pageId)->getSiteHash());
     $query->setFieldList('*');
     $query->setSorting('type asc, title asc');
     $this->search->search($query, 0, 10000);
     return $this->search->getResultDocumentsEscaped();
 }
Esempio n. 22
0
 /**
  * Gets all connection configurations for a given site.
  *
  * @param Site $site A TYPO3 site
  * @return array An array of Solr connection configurations for a site
  */
 public function getConfigurationsBySite(Site $site)
 {
     $solrConfigurations = array();
     $allConfigurations = $this->getAllConfigurations();
     foreach ($allConfigurations as $configuration) {
         if ($configuration['rootPageUid'] == $site->getRootPageId()) {
             $solrConfigurations[] = $configuration;
         }
     }
     return $solrConfigurations;
 }
Esempio n. 23
0
 /**
  *
  * @dataProvider canResolveAbsRefPrefixDataProvider
  * @param string $absRefPrefix
  * @param string $expectedUrl
  * @test
  */
 public function canResolveAbsRefPrefix($absRefPrefix, $expectedUrl)
 {
     $this->cleanUpSolrServerAndAssertEmpty();
     // create fake extension database table and TCA
     $this->importDumpFromFixture('fake_extension2_table.sql');
     $GLOBALS['TCA']['tx_fakeextension_domain_model_bar'] = (include $this->getFixturePath('fake_extension2_bar_tca.php'));
     $GLOBALS['TCA']['tx_fakeextension_domain_model_directrelated'] = (include $this->getFixturePath('fake_extension2_directrelated_tca.php'));
     $this->importDataSetFromFixture('can_index_custom_record_absRefPrefix_' . $absRefPrefix . '.xml');
     $this->addToIndexQueue('tx_fakeextension_domain_model_bar', 111);
     /** @var  $cliEnvironment CliEnvironment */
     $cliEnvironment = GeneralUtility::makeInstance(CliEnvironment::class);
     $cliEnvironment->backup();
     $cliEnvironment->initialize(PATH_site);
     /** @var $indexService IndexService */
     $site = Site::getFirstAvailableSite();
     $indexService = GeneralUtility::makeInstance(IndexService::class, $site);
     // run the indexer
     $indexService->indexItems(1);
     $cliEnvironment->restore();
     // do we have the record in the index with the value from the mm relation?
     sleep(2);
     $solrContent = file_get_contents('http://localhost:8080/solr/core_en/select?q=*:*');
     $this->assertContains('"numFound":1', $solrContent, 'Could not index document into solr');
     $this->assertContains('"url":"' . $expectedUrl . '"', $solrContent, 'Generated unexpected url with absRefPrefix = auto');
     $this->assertNotContains('auto', $solrContent, 'absRefPrefix=auto was not resolved');
     $this->cleanUpSolrServerAndAssertEmpty();
 }
Esempio n. 24
0
 /**
  * Gets $limit number of items to index for a particular $site.
  *
  * @param Site $site TYPO3 site
  * @param integer $limit Number of items to get from the queue
  * @return Item[] Items to index to the given solr server
  */
 public function getItemsToIndex(Site $site, $limit = 50)
 {
     $itemsToIndex = array();
     // determine which items to index with this run
     $indexQueueItemRecords = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows('*', 'tx_solr_indexqueue_item', 'root = ' . $site->getRootPageId() . ' AND changed > indexed' . ' AND changed <= ' . time() . ' AND errors = \'\'', '', 'indexing_priority DESC, changed DESC, uid DESC', intval($limit));
     if (!empty($indexQueueItemRecords)) {
         // convert queued records to index queue item objects
         $itemsToIndex = $this->getIndexQueueItemObjectsFromRecords($indexQueueItemRecords);
     }
     return $itemsToIndex;
 }
 /**
  * Checks any additional data that is relevant to this task. If the task
  * class is not relevant, the method is expected to return TRUE
  *
  * @param array $submittedData reference to the array containing the data submitted by the user
  * @param SchedulerModuleController $schedulerModule reference to the calling object (Scheduler's BE module)
  * @return bool True if validation was ok (or selected class is not relevant), FALSE otherwise
  */
 public function validateAdditionalFields(array &$submittedData, SchedulerModuleController $schedulerModule)
 {
     $result = false;
     // validate site
     $sites = Site::getAvailableSites();
     if (array_key_exists($submittedData['site'], $sites)) {
         $result = true;
     }
     return $result;
 }
 /**
  * Initializes resources commonly needed for several actions
  *
  * @return void
  */
 protected function initializeAction()
 {
     try {
         $site = $this->request->getArgument('site');
         if (is_numeric($site)) {
             $siteRootPageId = $this->request->getArgument('site');
             $this->site = Site::getSiteByPageId($siteRootPageId);
         } else {
             if ($site instanceof Site) {
                 $this->site = $site;
             }
         }
     } catch (NoSuchArgumentException $nsae) {
         $sites = Site::getAvailableSites();
         $site = array_shift($sites);
         $this->site = $site;
     }
     $this->request->setArgument('site', $this->site);
     $moduleData = $this->moduleDataStorageService->loadModuleData();
     $moduleData->setSite($this->site);
     $this->moduleDataStorageService->persistModuleData($moduleData);
 }
Esempio n. 27
0
 /**
  * @test
  */
 public function canGetAllSites()
 {
     $this->importDataSetFromFixture('can_get_all_sites.xml');
     $sites = Site::getAvailableSites();
     $this->assertSame(1, count($sites), 'Expected to retrieve one site from fixture');
 }
Esempio n. 28
0
 /**
  * @test
  */
 public function solrIsEmptyAfterCleanup()
 {
     $this->importDataSetFromFixture('can_reindex_task_fill_queue.xml');
     // fill the solr
     $site = Site::getFirstAvailableSite();
     $this->indexQueue->updateItem('pages', 1);
     $items = $this->indexQueue->getItems('pages', 1);
     /** @var $indexer \ApacheSolrForTypo3\Solr\IndexQueue\Indexer */
     $indexer = GeneralUtility::makeInstance('ApacheSolrForTypo3\\Solr\\IndexQueue\\Indexer');
     $indexer->index($items[0]);
     sleep(2);
     $this->assertSolrContainsDocumentCount(1);
     $this->task->setSite($site);
     $this->task->setIndexingConfigurationsToReIndex(array('pages'));
     $this->task->execute();
     sleep(2);
     // after the task was running the solr server should be empty
     $this->assertSolrIsEmpty();
     // if not we cleanup now
     $this->cleanUpSolrServerAndAssertEmpty();
 }
Esempio n. 29
0
 /**
  * Adds a page to the Index Queue of a site mounting the page.
  *
  * @param integer $mountedPageId ID (uid) of the mounted page.
  * @param array $mountProperties Array of mount point properties mountPageSource, mountPageDestination, and mountPageOverlayed
  */
 protected function addPageToMountingSiteIndexQueue($mountedPageId, array $mountProperties)
 {
     $mountingSite = Site::getSiteByPageId($mountProperties['mountPageDestination']);
     $pageInitializer = GeneralUtility::makeInstance('ApacheSolrForTypo3\\Solr\\IndexQueue\\Initializer\\Page');
     $pageInitializer->setSite($mountingSite);
     $pageInitializer->initializeMountedPage($mountProperties, $mountedPageId);
 }
Esempio n. 30
0
 /**
  * Finds the alternative page language overlay records for a page based on
  * the sys_language_mode.
  *
  * Possible Language Modes:
  * 1) content_fallback --> all languages
  * 2) strict --> available languages with page overlay
  * 3) ignore --> available languages with page overlay
  * 4) unknown mode or blank --> all languages
  *
  * @param integer $pageId Page ID.
  * @return array An array of translation overlays (or fake overlays) found for the given page.
  */
 protected function getTranslationOverlaysForPage($pageId)
 {
     $translationOverlays = array();
     $pageId = intval($pageId);
     $site = Site::getSiteByPageId($pageId);
     $languageModes = array('content_fallback', 'strict', 'ignore');
     $hasOverlayMode = in_array($site->getSysLanguageMode(), $languageModes, TRUE);
     $isContentFallbackMode = $site->getSysLanguageMode() === 'content_fallback';
     if ($hasOverlayMode && !$isContentFallbackMode) {
         $translationOverlays = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows('pid, sys_language_uid', 'pages_language_overlay', 'pid = ' . $pageId . BackendUtility::deleteClause('pages_language_overlay') . BackendUtility::BEenableFields('pages_language_overlay'));
     } else {
         // ! If no sys_language_mode is configured, all languages will be indexed !
         $languages = BackendUtility::getSystemLanguages();
         // remove default language (L = 0)
         array_shift($languages);
         foreach ($languages as $language) {
             $translationOverlays[] = array('pid' => $pageId, 'sys_language_uid' => $language[1]);
         }
     }
     return $translationOverlays;
 }