public function execute($debug) { $startTime = microtime(true); foreach (Kwf_Util_Fulltext_Backend_Abstract::getInstance()->getSubroots() as $subroot) { $t = time(); if ($debug) { echo "\n[{$subroot}] check-for-invalid...\n"; } $cmd = Kwf_Config::getValue('server.phpCli') . " bootstrap.php fulltext check-for-invalid-subroot --subroot={$subroot}"; if ($debug) { $cmd .= " --debug"; } //if ($this->_getParam('silent')) $cmd .= " --silent"; passthru($cmd, $ret); if ($ret) { exit($ret); } if ($debug) { echo "[{$subroot}] check-for-invalid finished: " . Kwf_View_Helper_SecondsAsDuration::secondsAsDuration(time() - $t) . "\n\n"; } $t = time(); if ($debug) { echo "\n[{$subroot}] check-contents...\n"; } $cmd = Kwf_Config::getValue('server.phpCli') . " bootstrap.php fulltext check-contents-subroot --subroot={$subroot}"; if (isset($this->_options['skipDiff'])) { $cmd .= " --skip-diff"; } if ($debug) { $cmd .= " --debug"; } //if ($this->_getParam('silent')) $cmd .= " --silent"; passthru($cmd, $ret); if ($ret) { exit($ret); } if ($debug) { echo "[{$subroot}] check-contents finished: " . Kwf_View_Helper_SecondsAsDuration::secondsAsDuration(time() - $t) . "\n\n"; } $t = time(); if ($debug) { echo "\n[{$subroot}] optimize...\n"; } Kwf_Util_Fulltext_Backend_Abstract::getInstance()->optimize($debug); if ($debug) { echo "[{$subroot}] optimize finished: " . Kwf_View_Helper_SecondsAsDuration::secondsAsDuration(time() - $t) . "\n\n"; } } if ($debug) { echo "\ncomplete fulltext check-contents finished: " . Kwf_View_Helper_SecondsAsDuration::secondsAsDuration(microtime(true) - $startTime) . "s\n"; } }
private function _query($select) { $limitCount = $select->getPart(Kwf_Model_Select::LIMIT_COUNT); $limitOffset = $select->getPart(Kwf_Model_Select::LIMIT_OFFSET); if ($limitOffset === null) { $limitOffset = 0; } $queryString = ''; $params = array(); if ($select->getPart(Kwf_Model_Select::WHERE_EXPRESSION)) { foreach ($select->getPart(Kwf_Model_Select::WHERE_EXPRESSION) as $exp) { if ($exp instanceof Kwf_Model_Select_Expr_SearchLike) { foreach ($exp->getSearchValues() as $field => $value) { if ($field == 'query') { $queryString = $value; } } } else { $query = $this->_getSearchQueryByExpr($exp); if (isset($params['fq'])) { $params['fq'] .= '+' . $query; } else { $params['fq'] = $query; } } } } if ($select->getPart(Kwf_Model_Select::WHERE_NOT_EQUALS)) { foreach ($select->getPart(Kwf_Model_Select::WHERE_NOT_EQUALS) as $field => $value) { if (isset($params['fq'])) { //if more than one fields should be queried they are apended with + $params['fq'] .= '+' . '-' . $field . ':' . $value; } else { //format is: -field:value for negation $params['fq'] = '-' . $field . ':' . $value; } } } if ($id = $select->getPart(Kwf_Model_Select::WHERE_ID)) { throw new Kwf_Exception_NotYetImplemented(); } $subroot = Kwf_Component_Data_Root::getInstance(); if ($select->getPart(Kwf_Model_Select::WHERE_EQUALS)) { foreach ($select->getPart(Kwf_Model_Select::WHERE_EQUALS) as $field => $value) { if ($field == 'subroot') { $subroot = $value; } } } $res = Kwf_Util_Fulltext_Backend_Abstract::getInstance()->userSearch($subroot, $queryString, $limitOffset, $limitCount, $params); if ($res['error']) { throw new Kwf_Exception("Fulltext error: " . $res['error']); } return $res; }
public function checkPagesSubrootAction() { Kwf_Util_MemoryLimit::set(256); $pageClassesThatCanHaveFulltext = array(); foreach (self::_getAllPossiblePageComponentClasses() as $c) { if (self::_canHaveFulltext($c)) { $pageClassesThatCanHaveFulltext[] = $c; } } $startTime = microtime(true); if (!$this->_getParam('componentId')) { throw new Kwf_Exception_Client("componentId parameter required"); } $componentId = $this->_getParam('componentId'); $queue = array($componentId); $stats = array('pages' => 0, 'indexedPages' => 0); while ($queue) { if (!$queue) { break; } if ($this->_getParam('debug')) { echo "queued: " . count($queue) . ' :: ' . round(memory_get_usage() / 1024 / 1024, 2) . "MB\n"; } $componentId = array_shift($queue); $stats['pages']++; //if ($this->_getParam('debug')) echo "==> ".$componentId; $page = Kwf_Component_Data_Root::getInstance()->getComponentById($componentId); //if ($this->_getParam('debug')) echo " :: $page->url\n"; if (!$page) { if ($this->_getParam('debug')) { echo "{$componentId} not found!\n"; } continue; } //echo "$page->url\n"; if ($this->_getParam('verbose')) { echo "getting child pages..."; } $childPages = $page->getChildPseudoPages(array('pageGenerator' => false, 'componentClasses' => $pageClassesThatCanHaveFulltext), array('pseudoPage' => false)); $childPages = array_merge($childPages, $page->getChildPseudoPages(array('pageGenerator' => true), array('pseudoPage' => false))); if ($this->_getParam('verbose')) { echo " done\n"; } foreach ($childPages as $c) { $i = $c; do { if (Kwc_Abstract::getFlag($i->componentClass, 'skipFulltextRecursive')) { continue 2; } } while ($i = $i->parent); if ($this->_getParam('verbose')) { echo "queued {$c->componentId}\n"; } $queue[] = $c->componentId; } unset($c); $hasFulltext = false; if (!Kwc_Abstract::getFlag($page->componentClass, 'skipFulltext') && $page->isPage && ($page->getRecursiveChildComponents(array('flag' => 'hasFulltext', 'inherit' => false)) || Kwc_Abstract::getFlag($page->componentClass, 'hasFulltext'))) { $hasFulltext = true; } if ($hasFulltext) { if (!Kwf_Util_Fulltext_Backend_Abstract::getInstance()->documentExists($page)) { if (Kwf_Util_Fulltext_Backend_Abstract::getInstance()->indexPage($page, $this->_getParam('debug'))) { $stats['indexedPages']++; if (!$this->_getParam('silent')) { echo "not found in index: {$page->componentId} has content!!!!\n"; } } else { if ($this->_getParam('debug')) { echo "not found in index: {$page->componentId} has NO content (that's ok)\n"; } } } } unset($page); if ($stats['pages'] % 50 == 0) { Kwf_Component_Data_Root::getInstance()->freeMemory(); } } if (!$this->_getParam('silent')) { echo "processed pages: {$stats['pages']}\n"; echo "indexed pages: {$stats['indexedPages']}\n"; } exit; }
public function checkContentsSubrootAction() { Kwf_Util_MemoryLimit::set(256); $subroot = Kwf_Component_Data_Root::getInstance()->getComponentById($this->_getParam('subroot')); if (!$subroot) { $subroot = Kwf_Component_Data_Root::getInstance(); } $i = 0; $stats = array('indexedPages' => 0, 'diffPages' => 0); $pagesMetaModel = Kwf_Component_PagesMetaModel::getInstance(); $select = new Kwf_Model_Select(); $select->whereEquals('deleted', false); $select->whereEquals('fulltext_skip', false); $select->whereEquals('subroot_component_id', $subroot->componentId); $it = new Kwf_Model_Iterator_Packages(new Kwf_Model_Iterator_Rows($pagesMetaModel, $select)); if ($this->_getParam('debug')) { $it = new Kwf_Iterator_ConsoleProgressBar($it); } foreach ($it as $row) { $componentId = $row->page_id; $page = Kwf_Component_Data_Root::getInstance()->getComponentById($componentId); if (!$this->_getParam('skip-diff')) { $docContent = Kwf_Util_Fulltext_Backend_Abstract::getInstance()->getDocumentContent($page); } else { $docContent = ''; } $newDoc = Kwf_Util_Fulltext_Backend_Abstract::getInstance()->getFulltextContentForPage($page); if (!$newDoc) { //this can happen (if there is no content) Kwf_Util_Fulltext_Backend_Abstract::getInstance()->deleteDocument($subroot, $componentId); $row = $pagesMetaModel->getRow($componentId); if ($row) { $row->delete(); } continue; } if (trim($newDoc['content']) != trim($docContent)) { $stats['diffPages']++; if (Kwf_Util_Fulltext_Backend_Abstract::getInstance()->indexPage($page)) { $stats['indexedPages']++; } if (!$this->_getParam('silent') && !$this->_getParam('skip-diff')) { echo "DIFF: {$componentId}\n"; } } unset($page); if ($i++ % 10) { Kwf_Component_Data_Root::getInstance()->freeMemory(); } //if ($this->_getParam('debug')) echo "memory_usage ".(memory_get_usage()/(1024*1024))."MB\n"; } if ($this->_getParam('debug')) { if (!$this->_getParam('skip-diff')) { echo "pages with diff: {$stats['diffPages']}\n"; } echo "indexed pages: {$stats['indexedPages']}\n"; } exit; }